From 0c37220868dd9d4cb1c8907a751c09cb2d1f9882 Mon Sep 17 00:00:00 2001 From: Macsim Date: Mon, 4 Aug 2025 14:34:56 +0300 Subject: [PATCH] some fixes --- .github/workflows/ci.yml | 29 +++++++++++++++--------- clone_repos.py | 48 ++++++++++++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6536a07..408021f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,6 @@ env: jobs: before: runs-on: ubuntu-latest - # Run on schedule or manual dispatch if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' steps: - name: Configure Git for github-actions[bot] @@ -58,7 +57,6 @@ jobs: update: runs-on: ubuntu-latest - # Run on schedule or manual dispatch if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' needs: before steps: @@ -67,6 +65,8 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git config --global --list + - name: Configure Git to ignore file mode changes + run: git config core.fileMode false - name: Checkout repository uses: actions/checkout@v4 with: @@ -78,6 +78,10 @@ jobs: python-version: '3.9' - name: Install Python dependencies run: pip install requests + - name: Debug file changes + run: | + git status + git diff --stat - name: Clone and update repositories env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -92,7 +96,6 @@ jobs: parse: runs-on: ubuntu-latest - # Run on schedule, manual dispatch, push to main, or merged PR if: | github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || @@ -105,11 +108,13 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git config --global --list + - name: Configure Git to ignore file mode changes + run: git config core.fileMode false - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: ${{ env.GIT_DEPTH }} - ref: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && env.BRANCH_NAME || 'main' }} + ref: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && env.BRANCH_NAME || 'main' }} - name: Set up Python 3.9 uses: actions/setup-python@v5 with: @@ -120,8 +125,8 @@ jobs: run: | echo "Running parse after update..." git fetch origin - git checkout ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && env.BRANCH_NAME || 'main' }} - git reset --hard origin/${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && env.BRANCH_NAME || 'main' }} + git checkout ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && env.BRANCH_NAME || 'main' }} + git reset --hard origin/${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && env.BRANCH_NAME || 'main' }} python3 -m venv venv source venv/bin/activate pip install --upgrade pip @@ -131,11 +136,10 @@ jobs: git add modules.json git commit -m "Updated modules.json after parse $(date +'%Y-%m-%d %H:%M:%S')" || echo "No changes for modules.json" git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@${REPO_URL}" - git push origin ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && env.BRANCH_NAME || 'main' }} + git push origin ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && env.BRANCH_NAME || 'main' }} commit: runs-on: ubuntu-latest - # Run on schedule or manual dispatch if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' needs: parse steps: @@ -144,11 +148,17 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git config --global --list + - name: Configure Git to ignore file mode changes + run: git config core.fileMode false - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: ${{ env.GIT_DEPTH }} ref: ${{ env.BRANCH_NAME }} + - name: Debug file changes + run: | + git status + git diff --stat - name: Final commit env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -161,7 +171,6 @@ jobs: create_pr: runs-on: ubuntu-latest - # Run on schedule or manual dispatch if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' needs: commit steps: @@ -180,7 +189,6 @@ jobs: run: | echo "Checking branch status before PR..." git fetch origin - # Check if the branch exists in the remote repository if git ls-remote --exit-code --heads origin ${{ env.BRANCH_NAME }} >/dev/null 2>&1; then echo "Branch ${{ env.BRANCH_NAME }} exists in remote repository." git log origin/${{ env.BRANCH_NAME }} -1 || echo "Failed to view log for ${{ env.BRANCH_NAME }}" @@ -193,7 +201,6 @@ jobs: backup: runs-on: ubuntu-latest - # Run on main branch or manual dispatch if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' needs: parse steps: diff --git a/clone_repos.py b/clone_repos.py index b8496d5..c606088 100644 --- a/clone_repos.py +++ b/clone_repos.py @@ -51,7 +51,21 @@ repos = [ "https://github.com/fiksofficial/python-modules" ] +def configure_git(): + """Configure Git to ignore file mode changes.""" + try: + subprocess.run( + ["git", "config", "core.fileMode", "false"], + check=True, + capture_output=True, + text=True, + ) + print("Git configured to ignore file mode changes.") + except subprocess.CalledProcessError as e: + print(f"Error configuring Git: {e.stderr}") + def is_repo_public(repo_url): + """Check if the repository is accessible.""" result = subprocess.run( ["git", "ls-remote", repo_url], stdout=subprocess.DEVNULL, @@ -60,6 +74,7 @@ def is_repo_public(repo_url): return result.returncode == 0 def is_repo_accessible(repo_url): + """Check if the repository URL is accessible via HTTP.""" try: response = requests.head(repo_url, timeout=5) return response.status_code == 200 @@ -67,45 +82,52 @@ def is_repo_accessible(repo_url): return False def is_valid_filename(filename): + """Check if the filename contains invalid characters.""" invalid_chars = r'[<>:"/\\|?*]' return not re.search(invalid_chars, filename) def rename_invalid_files(local_path): + """Rename files with invalid characters in their names.""" for root, dirs, files in os.walk(local_path): for file in files: if not is_valid_filename(file): old_path = os.path.join(root, file) new_file = re.sub(r'[<>:"/\\|?*]', "_", file) new_path = os.path.join(root, new_file) - os.rename(old_path, new_path) - print(f"Переименован файл: {old_path} -> {new_path}") + try: + os.rename(old_path, new_path) + print(f"Renamed file: {old_path} -> {new_path}") + except OSError as e: + print(f"Error renaming file {old_path}: {e}") def get_repo_path(repo_url): + """Extract the repository path from the URL.""" return repo_url.replace("https://github.com/", "") def clean_unused_repos(): + """Remove directories not in the repos list or for inaccessible repositories.""" current_dir = os.getcwd() - print(f"Текущая директория: {current_dir}") + print(f"Current directory: {current_dir}") existing_dirs = { d for d in os.listdir(current_dir) if os.path.isdir(os.path.join(current_dir, d)) } - print(f"Все директории до фильтрации: {existing_dirs}") + print(f"All directories before filtering: {existing_dirs}") protected_dirs = {".git", ".github", "assets"} existing_dirs.difference_update(protected_dirs) - print(f"Директории после исключения защищённых: {existing_dirs}") + print(f"Directories after excluding protected: {existing_dirs}") expected_dirs = {get_repo_path(url).split("/")[0] for url in repos} - print(f"Ожидаемые директории: {expected_dirs}") + print(f"Expected directories: {expected_dirs}") for dir_name in existing_dirs: dir_path = os.path.join(current_dir, dir_name) if dir_name not in expected_dirs: shutil.rmtree(dir_path, ignore_errors=True) - print(f"Удалена директория, отсутствующая в списке repos: {dir_path}") + print(f"Removed directory not in repos list: {dir_path}") for repo_url in repos: repo_path = get_repo_path(repo_url) @@ -113,9 +135,10 @@ def clean_unused_repos(): if os.path.exists(local_path): if not is_repo_accessible(repo_url): shutil.rmtree(local_path, ignore_errors=True) - print(f"Удалена директория недоступного или удалённого репозитория: {local_path}") + print(f"Removed directory for inaccessible repository: {local_path}") def clone_or_update_repo(repo_url): + """Clone or update a repository and process its files.""" repo_path = get_repo_path(repo_url) owner, repo_name = repo_path.split("/") local_path = os.path.join(owner, repo_name) @@ -125,10 +148,10 @@ def clone_or_update_repo(repo_url): if os.path.exists(local_path): shutil.rmtree(local_path) - print(f"Удалена старая директория: {local_path}") + print(f"Removed old directory: {local_path}") if not is_repo_public(repo_url): - print(f"Пропускаем закрытый или недоступный репозиторий: {repo_url}") + print(f"Skipping private or inaccessible repository: {repo_url}") return try: @@ -140,11 +163,12 @@ def clone_or_update_repo(repo_url): ) shutil.rmtree(os.path.join(local_path, ".git"), ignore_errors=True) rename_invalid_files(local_path) - print(f"Клонирован и обработан репозиторий: {repo_url} -> {local_path}") + print(f"Cloned and processed repository: {repo_url} -> {local_path}") except subprocess.CalledProcessError as e: - print(f"Ошибка при клонировании {repo_url}: {e.stderr}, пропускаем.") + print(f"Error cloning {repo_url}: {e.stderr}, skipping.") if __name__ == "__main__": + configure_git() # Set Git configuration at the start clean_unused_repos() for repo_url in repos: clone_or_update_repo(repo_url) \ No newline at end of file