Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions crates/crates_io_database/src/models/user.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ pub struct OauthGithub {
pub avatar: Option<String>,
/// In the process of being migrated from `users.gh_encrypted_token`.
pub encrypted_token: Vec<u8>,
/// The last time we verified with GitHub what the GitHub username for this user was, and
/// whether the account was valid
pub last_sync: DateTime<Utc>,
/// In the process of being migrated from `users.gh_login`.
pub login: String,
/// Foreign key to the `users` table.
Expand Down Expand Up @@ -193,6 +196,7 @@ impl NewOauthGithub<'_> {
oauth_github::encrypted_token.eq(excluded(oauth_github::encrypted_token)),
oauth_github::login.eq(excluded(oauth_github::login)),
oauth_github::avatar.eq(excluded(oauth_github::avatar)),
oauth_github::last_sync.eq(Utc::now()),
))
.get_result(&mut conn)
.await
Expand Down
2 changes: 2 additions & 0 deletions crates/crates_io_database/src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,8 @@ diesel::table! {
avatar -> Nullable<Varchar>,
/// Encrypted GitHub access token
encrypted_token -> Bytea,
/// The last time we verified with GitHub what the GitHub username for this user was, and whether the account was valid
last_sync -> Timestamptz,
/// GitHub username
login -> Varchar,
/// Crates.io user ID foreign key
Expand Down
1 change: 1 addition & 0 deletions crates/crates_io_database_dump/src/dump-db.toml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ account_id = "private"
encrypted_token = "private"
login = "private"
avatar = "private"
last_sync = "private"
[oauth_github.column_defaults]
encrypted_token = "''"

Expand Down
7 changes: 7 additions & 0 deletions crates/crates_io_github/examples/test_github_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ enum Request {
#[clap(long, env = "GITHUB_ACCESS_TOKEN", hide_env_values = true)]
access_token: SecretString,
},
GetUserById {
account_id: i64,
},
OrgByName {
org_name: String,
#[clap(long, env = "GITHUB_ACCESS_TOKEN", hide_env_values = true)]
Expand Down Expand Up @@ -68,6 +71,10 @@ async fn main() -> Result<()> {
let response = github_client.get_user(&name, &access_token).await?;
println!("{response:#?}");
}
Request::GetUserById { account_id } => {
let response = github_client.get_user_by_id(account_id).await?;
println!("{response:#?}");
}
Request::OrgByName {
org_name,
access_token,
Expand Down
6 changes: 6 additions & 0 deletions crates/crates_io_github/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type Result<T> = std::result::Result<T, GitHubError>;
pub trait GitHubClient: Send + Sync {
async fn current_user(&self, auth: &AccessToken) -> Result<GitHubUser>;
async fn get_user(&self, name: &str, auth: &AccessToken) -> Result<GitHubUser>;
async fn get_user_by_id(&self, account_id: i64) -> Result<GitHubUser>;
async fn org_by_name(&self, org_name: &str, auth: &AccessToken) -> Result<GitHubOrganization>;
async fn team_by_name(
&self,
Expand Down Expand Up @@ -213,6 +214,11 @@ impl GitHubClient for RealGitHubClient {
self.request(&url, auth).await
}

async fn get_user_by_id(&self, account_id: i64) -> Result<GitHubUser> {
let url = format!("/user/{account_id}");
self._request(&url, std::convert::identity).await
}

async fn org_by_name(&self, org_name: &str, auth: &AccessToken) -> Result<GitHubOrganization> {
let url = format!("/orgs/{org_name}");
self.request(&url, auth).await
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
ALTER TABLE IF EXISTS oauth_github
DROP COLUMN last_sync;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
-- safety-assured:start
-- Adding a column with a constant value default is safe because we're using Postgres > 11.
ALTER TABLE IF EXISTS oauth_github
-- Set existing rows' last sync value to 1970.
ADD COLUMN last_sync timestamptz NOT NULL DEFAULT to_timestamp(0);
-- safety-assured:end

comment on column oauth_github.last_sync is 'The last time we verified with GitHub what the GitHub username for this user was, and whether the account was valid';
44 changes: 43 additions & 1 deletion src/bin/crates-admin/enqueue_job.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use anyhow::Result;
use chrono::NaiveDate;
use crates_io::db;
use crates_io::schema::{background_jobs, crates};
use crates_io::models::OauthGithub;
use crates_io::schema::{background_jobs, crates, oauth_github};
use crates_io::worker::jobs;
use crates_io_worker::BackgroundJob;
use diesel::dsl::exists;
Expand Down Expand Up @@ -56,6 +57,14 @@ pub enum Command {
SyncUpdatesFeed,
TrustpubCleanup,
UpdateDownloads,
/// Sync the oldest batch of users with GitHub
UpdateUserBatch {
#[arg(long = "dry-run")]
dry_run: bool,

#[arg(long = "batch-size", default_value = "100")]
batch_size: usize,
},
}

pub async fn run(command: Command) -> Result<()> {
Expand Down Expand Up @@ -169,6 +178,39 @@ pub async fn run(command: Command) -> Result<()> {
jobs::UpdateDownloads.enqueue(&conn).await?;
}
}

Command::UpdateUserBatch {
dry_run,
batch_size,
} => {
let oldest_oauth_github_records = oauth_github::table
.order(oauth_github::last_sync.asc())
.limit(batch_size as i64)
.load::<OauthGithub>(&mut conn)
.await?;

for oauth_github in oldest_oauth_github_records {
let user_id = oauth_github.user_id;
let github_id = oauth_github.account_id;
let old_username = oauth_github.login.clone();

let job = jobs::UpdateUserFromGithub {
dry_run,
account_id: oauth_github.account_id,
};

// Don't stop the whole batch if one user update errors, but do log the error
if let Err(e) = job.enqueue(&conn).await {
error!(
error = %e,
user_id,
github_id,
old_username,
"Error running UpdateUserFromGithub"
);
}
}
}
Comment thread
carols10cents marked this conversation as resolved.
};

Ok(())
Expand Down
18 changes: 18 additions & 0 deletions src/tests/util/github.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ impl MockData {
mock.expect_current_user()
.returning(|_auth| self.current_user());

mock.expect_get_user_by_id()
.returning(|account_id| self.get_user_by_id(account_id));

mock.expect_org_by_name()
.returning(|org_name, _auth| self.org_by_name(org_name));

Expand Down Expand Up @@ -81,6 +84,21 @@ impl MockData {
})
}

fn get_user_by_id(&self, account_id: i64) -> Result<GitHubUser, GitHubError> {
let user = self
.users
.iter()
.find(|user| user.id as i64 == account_id)
.ok_or_else(not_found)?;
Ok(GitHubUser {
id: user.id,
login: user.login.into(),
name: Some(user.name.into()),
email: Some(user.email.into()),
avatar_url: Some(format!("https://avatars.example.com/{}", user.id)),
})
}

fn org_by_name(&self, org_name: &str) -> Result<GitHubOrganization, GitHubError> {
let org = self
.orgs
Expand Down
1 change: 1 addition & 0 deletions src/tests/worker/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ mod squash_index;
mod sync_admins;
mod trustpub;
mod update_default_version;
mod update_user_from_github;
Loading