Skip to content

Commit 4bbb318

Browse files
scholtzanscholtzan
committed
initial migration
1 parent 2115724 commit 4bbb318

24 files changed

Lines changed: 1412 additions & 33 deletions

‎.github/workflows/build.yml‎

Lines changed: 1167 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/setup-main.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
name: Main Branch Workflow
2+
on:
3+
push:
4+
branches:
5+
6+
jobs:
7+
main-logic:
8+
uses: ./.github/workflows/workflows.yml
9+
with:
10+
deploy: true

‎.github/workflows/setup-pr.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Setup PR Validations
2+
3+
on:
4+
pull_request:
5+
branches:
6+
7+
permissions:
8+
contents: read
9+
10+
jobs:
11+
filter:
12+
runs-on: ubuntu-latest
13+
outputs:
14+
validate_bqetl: ${{ steps.bqetl_files.outputs.any_changed }}
15+
validate_sql: ${{ steps.sql_files.outputs.any_changed }}
16+
validate_routines: ${{ steps.routines_files.outputs.any_changed }}
17+
trigger_sql_generation: ${{ steps.sql_gen_files.outputs.any_changed }}
18+
steps:
19+
- name: Checkout code
20+
uses: actions/checkout@v4
21+
with:
22+
fetch-depth: 0
23+
persist-credentials: false
24+
25+
- name: Get changed bqetl files
26+
id: bqetl_files
27+
uses: tj-actions/changed-files@v44
28+
with:
29+
files: |
30+
requirements.txt
31+
bigquery_etl/**
32+
tests/**
33+
script/bqetl
34+
script/entrypoint
35+
.circleci/**
36+
37+
- name: Get changed sql files
38+
id: sql_files
39+
uses: tj-actions/changed-files@v44
40+
with:
41+
files: |
42+
requirements.txt
43+
bigquery_etl/query_scheduling/**
44+
sql/**
45+
sql_generators/**
46+
tests/sql/**
47+
bqetl_project.yaml
48+
dags.yaml
49+
.circleci/**
50+
51+
- name: Get changed routines files
52+
id: routines_files
53+
uses: tj-actions/changed-files@v44
54+
with:
55+
files: |
56+
requirements.txt
57+
bigquery_etl/routine/**
58+
sql/mozfun/**
59+
sql/moz-fx-data-shared-prod/udf/**
60+
sql/moz-fx-data-shared-prod/udf_js/**
61+
bqetl_project.yaml
62+
.circleci/**
63+
64+
- name: Get changed sql-generation files
65+
id: sql_gen_files
66+
uses: tj-actions/changed-files@v44
67+
with:
68+
files: |
69+
requirements.txt
70+
bqetl_project.yaml
71+
72+
validate-bqetl:
73+
needs: filter
74+
if: needs.filter.outputs.validate_bqetl == 'true'
75+
uses: ./.github/workflows/validate-bqetl.yml
76+
secrets: inherit
77+
78+
validate-sql:
79+
needs: filter
80+
if: needs.filter.outputs.validate_sql == 'true'
81+
uses: ./.github/workflows/validate-sql.yml
82+
secrets: inherit
83+
84+
validate-routines:
85+
needs: filter
86+
if: needs.filter.outputs.validate_routines == 'true'
87+
uses: ./.github/workflows/validate-routines.yml
88+
secrets: inherit
89+
90+
trigger-sql-generation:
91+
needs: filter
92+
if: needs.filter.outputs.trigger_sql_generation == 'true'
93+
uses: ./.github/workflows/trigger-sql-generation.yml
94+
secrets: inherit

‎.github/workflows/setup-tag.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# This workflow is triggered on tag pushes. It demonstrates a secure deployment to PyPI.
2+
# The original CircleCI configuration was a 'setup' workflow that triggered another
3+
# configuration file on tags. In GitHub Actions, this is handled by a single workflow
4+
# with a tag trigger.
5+
6+
name: Tagged Deploy
7+
8+
on:
9+
push:
10+
tags:
11+
12+
jobs:
13+
deploy:
14+
name: Deploy to PyPI
15+
if: github.event.base_ref == 'refs/heads/main'
16+
runs-on: ubuntu-latest
17+
permissions:
18+
id-token: write # Required for OIDC trusted publishing
19+
contents: read
20+
environment:
21+
name: pypi
22+
url: https://pypi.org/p/<package-name> # TODO: Replace <package-name> with your PyPI package name
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v4
26+
with:
27+
persist-credentials: false
28+
29+
- name: Set up Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: '3.10' # TODO: Use the project's actual Python version
33+
34+
- name: Install build dependencies
35+
run: python -m pip install --upgrade build
36+
37+
- name: Build distribution files
38+
run: python -m build --sdist
39+
40+
- name: Publish distribution to PyPI
41+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/tagged-deploy.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Deploy to PyPI
2+
3+
on:
4+
push:
5+
branches:
6+
paths:
7+
8+
jobs:
9+
check-version-change:
10+
name: Check for version change
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: read
14+
outputs:
15+
proceed: ${{ steps.version_change.outputs.proceed }}
16+
steps:
17+
- name: Checkout code
18+
uses: actions/checkout@v4
19+
with:
20+
fetch-depth: 0
21+
persist-credentials: false
22+
23+
- name: Check for package version change in last commit
24+
id: version_change
25+
run: |
26+
if git diff HEAD~1 HEAD -- pyproject.toml | grep -E '^\+version'; then
27+
echo "Found changes to package version, proceeding with deployment."
28+
echo "proceed=true" >> "$GITHUB_OUTPUT"
29+
else
30+
echo "No changes in package version. Skipping deployment."
31+
echo "proceed=false" >> "$GITHUB_OUTPUT"
32+
fi
33+
34+
deploy:
35+
name: Build and Deploy to PyPI
36+
needs: check-version-change
37+
if: needs.check-version-change.outputs.proceed == 'true'
38+
runs-on: ubuntu-latest
39+
permissions:
40+
id-token: write
41+
contents: read
42+
environment:
43+
name: pypi
44+
url: https://pypi.org/p/your-package-name
45+
steps:
46+
- name: Checkout code
47+
uses: actions/checkout@v4
48+
with:
49+
persist-credentials: false
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v5
53+
with:
54+
python-version: '3.10'
55+
56+
- name: Install build dependencies
57+
run: python -m pip install --upgrade build
58+
59+
- name: Build distribution files
60+
run: python -m build --sdist --wheel
61+
62+
- name: Publish distribution to PyPI
63+
uses: pypa/gh-action-pypi-publish@release/v1

‎.yamllint.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ rules:
1313
ignore: |
1414
.git/
1515
venv/
16+
.venv/
1617
bigquery_etl/events_daily/
1718
sql_generators/

‎bigquery_etl/dryrun.py‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -71,16 +71,19 @@ def get_credentials(auth_req: Optional[GoogleAuthRequest] = None):
7171

7272
def get_id_token(dry_run_url=ConfigLoader.get("dry_run", "function"), credentials=None):
7373
"""Get token to authenticate against Cloud Function."""
74-
auth_req = GoogleAuthRequest()
75-
credentials = credentials or get_credentials(auth_req)
76-
77-
if hasattr(credentials, "id_token"):
78-
# Get token from default credentials for the current environment created via Cloud SDK run
79-
id_token = credentials.id_token
80-
else:
81-
# If the environment variable GOOGLE_APPLICATION_CREDENTIALS is set to service account JSON file,
82-
# then ID token is acquired using this service account credentials.
83-
id_token = fetch_id_token(auth_req, dry_run_url)
74+
# look for token created by the GitHub Actions workflow
75+
id_token = os.environ.get("GOOGLE_GHA_ID_TOKEN")
76+
77+
if not id_token:
78+
auth_req = GoogleAuthRequest()
79+
credentials = credentials or get_credentials(auth_req)
80+
if hasattr(credentials, "id_token"):
81+
# Get token from default credentials for the current environment created via Cloud SDK run
82+
id_token = credentials.id_token
83+
else:
84+
# If the environment variable GOOGLE_APPLICATION_CREDENTIALS is set to service account JSON file,
85+
# then ID token is acquired using this service account credentials.
86+
id_token = fetch_id_token(auth_req, dry_run_url)
8487
return id_token
8588

8689

‎tests/alchemer/test_survey.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ def test_insert_to_bq_subquestions(testing_table_id):
471471

472472
@pytest.mark.integration
473473
def test_cli(patch_api_requests, testing_table_id):
474-
res = CliRunner().invoke(
474+
res = CliRunner(env={"NO_COLOR": "1"}).invoke(
475475
main,
476476
[
477477
"--date",

‎tests/backfill/test_shredder_mitigation.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,7 @@ class TestSubset:
707707

708708
@pytest.fixture
709709
def runner(self):
710-
return CliRunner()
710+
return CliRunner(env={"NO_COLOR": "1"})
711711

712712
@patch("google.cloud.bigquery.Client")
713713
def test_version(self, mock_client):
@@ -985,7 +985,7 @@ class TestGenerateQueryWithShredderMitigation:
985985
@pytest.fixture
986986
def runner(self):
987987
"""Runner"""
988-
return CliRunner()
988+
return CliRunner(env={"NO_COLOR": "1"})
989989

990990
@patch("google.cloud.bigquery.Client")
991991
@patch("bigquery_etl.backfill.shredder_mitigation.classify_columns")

‎tests/cli/test_cli_alchemer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ def nop(*args, **kwargs):
1010
monkeypatch.setattr("bigquery_etl.cli.alchemer.get_survey_data", nop)
1111
monkeypatch.setattr("bigquery_etl.cli.alchemer.insert_to_bq", nop)
1212

13-
result = CliRunner().invoke(
13+
result = CliRunner(env={"NO_COLOR": "1"}).invoke(
1414
backfill,
1515
[
1616
"--start-date",

0 commit comments

Comments
 (0)