mirror of
https://github.com/MuRuLOSE/limoka.git
synced 2026-06-16 22:34:19 +02:00
267 lines
11 KiB
YAML
267 lines
11 KiB
YAML
name: CI Pipeline
|
|
|
|
# Triggers for the workflow
|
|
on:
|
|
schedule:
|
|
- cron: '0 0 * * *' # Runs daily at midnight UTC
|
|
push:
|
|
branches:
|
|
- main
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
workflow_dispatch: # Allows manual triggering from GitHub UI
|
|
|
|
# Environment variables available to all jobs
|
|
env:
|
|
BRANCH_NAME: "update-submodules_${{ github.sha }}"
|
|
REPO_URL: "github.com/MuRuLOSE/limoka.git" # Repository URL
|
|
GIT_DEPTH: 0 # Fetch full history for Git operations
|
|
|
|
jobs:
|
|
before:
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Set up Python 3.9
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.9'
|
|
- name: Install Python dependencies
|
|
run: pip install requests
|
|
- name: Sync main branch and create new branch
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
git fetch origin
|
|
git checkout main
|
|
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${REPO_URL}"
|
|
echo "Synchronizing main with origin/main..."
|
|
git reset --hard origin/main
|
|
echo "Deleting remote branch ${BRANCH_NAME}..."
|
|
git push origin --delete ${BRANCH_NAME} || echo "Branch ${BRANCH_NAME} did not exist or could not be deleted"
|
|
echo "Deleting local branch ${BRANCH_NAME}..."
|
|
git branch -D ${BRANCH_NAME} || echo "Local branch ${BRANCH_NAME} did not exist"
|
|
git checkout -b ${BRANCH_NAME}
|
|
echo "Creating new branch ${BRANCH_NAME}..."
|
|
git push origin ${BRANCH_NAME} --force || echo "Error creating branch"
|
|
|
|
update:
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
needs: before
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
ref: ${{ env.BRANCH_NAME }}
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Configure Git to ignore file mode changes
|
|
run: git config core.fileMode false
|
|
- name: Set up Python 3.9
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.9'
|
|
- name: Install Python dependencies
|
|
run: pip install requests
|
|
- name: Debug file changes
|
|
run: |
|
|
git status
|
|
git diff --stat
|
|
- name: Clone and update repositories
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
echo "Cloning and updating repositories..."
|
|
python3 clone_repos.py || echo "Error running clone_repos.py, continuing with commit and push"
|
|
git add .
|
|
git commit -m "Added and updated repositories $(date +'%Y-%m-%d %H:%M:%S')" || echo "No changes for commit"
|
|
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${REPO_URL}"
|
|
git push origin ${{ env.BRANCH_NAME }} --force
|
|
|
|
parse:
|
|
runs-on: ubuntu-latest
|
|
if: |
|
|
github.event_name == 'schedule' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'push' && github.ref == 'refs/heads/main') ||
|
|
(github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true)
|
|
needs: update
|
|
steps:
|
|
- name: Set branch ref for parse
|
|
id: setref
|
|
run: |
|
|
if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
echo "ref=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "ref=main" >> $GITHUB_OUTPUT
|
|
fi
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
ref: ${{ steps.setref.outputs.ref }}
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Configure Git to ignore file mode changes
|
|
run: git config core.fileMode false
|
|
- name: Set up Python 3.9
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.9'
|
|
- name: Run parse scripts
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
echo "Running parse after update..."
|
|
git fetch origin
|
|
git checkout ${{ steps.setref.outputs.ref }}
|
|
git reset --hard origin/${{ steps.setref.outputs.ref }}
|
|
python3 -m venv venv
|
|
source venv/bin/activate
|
|
pip install --upgrade pip
|
|
pip install requests scikit-learn tqdm
|
|
python3 parse.py
|
|
python3 categories.py
|
|
git add modules.json
|
|
git commit -m "Updated modules.json after parse $(date +'%Y-%m-%d %H:%M:%S')" || echo "No changes for modules.json"
|
|
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${REPO_URL}"
|
|
git push origin ${{ steps.setref.outputs.ref }}
|
|
|
|
commit:
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
needs: parse
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
ref: ${{ env.BRANCH_NAME }}
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Configure Git to ignore file mode changes
|
|
run: git config core.fileMode false
|
|
- name: Debug file changes
|
|
run: |
|
|
git status
|
|
git diff --stat
|
|
- name: Final commit
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
git checkout ${{ env.BRANCH_NAME }}
|
|
git add . || echo "No changes"
|
|
git commit -m "Final commit $(date +'%Y-%m-%d %H:%M:%S')" || echo "No changes for final commit"
|
|
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${REPO_URL}"
|
|
git push origin ${{ env.BRANCH_NAME }} --force
|
|
|
|
create_pr:
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
needs: commit
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Create Pull Request
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
echo "Checking branch status before PR..."
|
|
git fetch origin
|
|
if git ls-remote --exit-code --heads origin ${{ env.BRANCH_NAME }} >/dev/null 2>&1; then
|
|
echo "Branch ${{ env.BRANCH_NAME }} exists in remote repository."
|
|
git log origin/${{ env.BRANCH_NAME }} -1 || echo "Failed to view log for ${{ env.BRANCH_NAME }}"
|
|
git diff origin/main origin/${{ env.BRANCH_NAME }} || echo "No diff between main and ${{ env.BRANCH_NAME }}"
|
|
echo "Creating Pull Request..."
|
|
gh pr create --base main --head ${{ env.BRANCH_NAME }} --title "Update of repositories $(date +'%Y-%m-%d %H:%M:%S')" --body "Automated PR for repository updates" || echo "Failed to create PR"
|
|
else
|
|
echo "Branch ${{ env.BRANCH_NAME }} does not exist in remote repository, skipping PR creation."
|
|
fi
|
|
|
|
backup:
|
|
runs-on: ubuntu-latest
|
|
if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
|
|
needs: parse
|
|
steps:
|
|
- name: Configure Git for github-actions[bot]
|
|
run: |
|
|
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --global user.name "github-actions[bot]"
|
|
git config --global --list
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: ${{ env.GIT_DEPTH }}
|
|
- name: Create and send backup to Telegram
|
|
env:
|
|
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
|
|
TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
|
|
run: |
|
|
echo "Creating .zip file of the repository with maximum compression..."
|
|
git archive --format=zip --output=repository-original.zip HEAD
|
|
zip -9 repository.zip repository-original.zip
|
|
rm repository-original.zip
|
|
echo "File size of the created .zip file:"
|
|
du -sh repository.zip
|
|
echo "Splitting the .zip file into 8 parts..."
|
|
split -b 49M repository.zip repository-part-
|
|
echo "Files created after split:"
|
|
ls repository-part-*
|
|
COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
|
|
COMMIT_DATE="$(date --date="$(git log -1 --pretty=%ci)" +'%Y-%m-%d %H:%M:%S')"
|
|
COMMIT_HASH="$(git rev-parse --short=6 HEAD)"
|
|
COMMIT_URL="https://${REPO_URL}/commit/$(git rev-parse HEAD)"
|
|
MESSAGE="Commit Date: $COMMIT_DATE, Commit Message: $COMMIT_MESSAGE, Commit Hash: [\`$COMMIT_HASH\`]($COMMIT_URL)"
|
|
|
|
echo "Sending .zip file parts to Telegram..."
|
|
|
|
FIRST_PART=true
|
|
for part in $(ls repository-part-* | sort); do
|
|
if $FIRST_PART; then
|
|
TELEGRAM_API_URL="https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendDocument"
|
|
echo "Sending first file to Telegram: $TELEGRAM_API_URL"
|
|
curl -X POST "$TELEGRAM_API_URL" \
|
|
-F chat_id=$TELEGRAM_CHAT_ID \
|
|
-F document=@$part \
|
|
-F caption="$MESSAGE" \
|
|
-F parse_mode="Markdown"
|
|
FIRST_PART=false
|
|
else
|
|
TELEGRAM_API_URL="https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendDocument"
|
|
echo "Sending file to Telegram: $TELEGRAM_API_URL"
|
|
curl -X POST "$TELEGRAM_API_URL" \
|
|
-F chat_id=$TELEGRAM_CHAT_ID \
|
|
-F document=@$part
|
|
fi
|
|
done
|
|
|
|
echo "Files sent to Telegram successfully!"
|