diff --git a/.github/workflows/nightly_check.yml b/.github/workflows/nightly_check.yml index dbb200c..6a4a87c 100644 --- a/.github/workflows/nightly_check.yml +++ b/.github/workflows/nightly_check.yml @@ -1,63 +1,75 @@ name: Nightly Release on: schedule: # 03:00 UTC ogni giorno - cron: "0 3 * * *" # opzionale: permette lancio manuale workflow_dispatch: permissions: contents: write jobs: build-and-release: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install dependencies run: | pip install -r requirements.txt - name: Run script run: | python scripts/generate_data.py - name: Commit versions.json + if: steps.check.outputs.changed == 'true' run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add public_data/versions.json # evita errore se non ci sono modifiche git diff --cached --quiet || git commit -m "Update versions.json" git push - name: Generate release tag id: tag + if: steps.check.outputs.changed == 'true' run: | - TAG="daily-$(date +'%Y%m%d-%H%M')" + TAG="data-$(date +'%Y%m%d')" echo "tag=$TAG" >> $GITHUB_OUTPUT - name: Create Release uses: softprops/action-gh-release@v2 + if: steps.check.outputs.changed == 'true' with: tag_name: ${{ steps.tag.outputs.tag }} name: Daily Release ${{ steps.tag.outputs.tag }} files: | public_data/versions.json public_data/*.txt.gz env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Cleanup old releases + if: steps.check.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release list --limit 100 --json tagName \ + | jq -r '.[10:] | .[].tagName' \ + | xargs -I {} sh -c 'gh release delete {} --yes --cleanup-tag' \ No newline at end of file diff --git a/parse_gtfs.py b/parse_gtfs.py index 15d36f0..af65262 100644 --- a/parse_gtfs.py +++ b/parse_gtfs.py @@ -1,82 +1,93 @@ import os import json import hashlib import requests import zipfile from datetime import date, datetime, timezone import io import gzip from pathlib import Path # Configurazione GTFS_URL = "https://www.gtt.to.it/open_data/gtt_gtfs.zip" OUTPUT_DIR = Path("public_data") VERSIONS_FILE = OUTPUT_DIR / "versions.json" def get_file_hash(file_bytes): """Calcola l'hash MD5 del contenuto scompattato.""" return hashlib.md5(file_bytes).hexdigest() + +def set_github_output(key, value): + path = os.environ.get("GITHUB_OUTPUT", "") + if path: + with open(path, "a") as f: + f.write(f"{key}={value}\n") + def main(compress:bool=True): os.makedirs(OUTPUT_DIR, exist_ok=True) if VERSIONS_FILE.exists(): with open(VERSIONS_FILE, 'r', encoding='utf-8') as f: versions = json.load(f) else: versions = {} print(f"Scaricando GTFS da {GTFS_URL}...") response = requests.get(GTFS_URL) response.raise_for_status() + changed_any = False with zipfile.ZipFile(io.BytesIO(response.content)) as z: for filename in z.namelist(): if not filename.endswith('.txt'): continue print(f"Elaborando {filename}...") file_bytes = z.read(filename) # L'hash viene calcolato sui byte RAW (scompattati) current_hash = get_file_hash(file_bytes) file_info = versions.get(filename, {}) changed = False timenow = datetime.now(timezone.utc).replace(microsecond=0) version_num = int(file_info.get("version_int", -1)) if file_info.get("hash") != current_hash: print(f" -> [AGGIORNATO] {filename} รจ cambiato.") file_info["hash"] = current_hash file_info["last_modified"] = timenow.isoformat() file_info["version_int"] = version_num+1 versions[filename] = file_info changed = True else: print(f" -> [INVARIATO] {filename}.") # Gestione speciale per la tabella molto grande #if filename == "connections.txt": if(compress): output_filename = filename + ".gz" output_path = os.path.join(OUTPUT_DIR, output_filename) print(f" -> Comprimo {filename} in formato Gzip...") with gzip.open(output_path, 'wb', compresslevel=6) as f_out: f_out.write(file_bytes) else: # Tutti gli altri file standard rimangono in formato testo libero output_path = os.path.join(OUTPUT_DIR, filename) with open(output_path, 'wb') as f: f.write(file_bytes) - + if(changed): changed_any = True + + + set_github_output("changed", str(changed_any)) # Salva il file delle versioni with open(VERSIONS_FILE, 'w', encoding='utf-8') as f: json.dump(versions, f, indent=4) print("Elaborazione e compressione completate.") if __name__ == "__main__": main(True) \ No newline at end of file