mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-06-01 08:48:46 +02:00
ef4c5eb65e
Please note we have force-pushed a new initial commit in order to remove some publicly-available Twitter user information. Note that this process may be required in the future.
37 lines
1.1 KiB
Plaintext
37 lines
1.1 KiB
Plaintext
# See
|
|
# Checkout the README to see how to deploy the job
|
|
|
|
class Profile(Struct):
|
|
project = Required(String)
|
|
date = Required(String)
|
|
environment = Default(String, 'dev')
|
|
machine= Default(String, 'n2-highmem-16')
|
|
|
|
job = Job(
|
|
name='assemble-multi-type-graph-scio-adhoc-app',
|
|
project='{{profile.project}}',
|
|
staging_bucket='{{profile.project}}',
|
|
service_account='{{profile.user_name}}-shdw@twttr-dp-svc-accounts.iam.gserviceaccount.com',
|
|
region='us-central1',
|
|
worker_config=WorkerConfig(
|
|
num_workers=2,
|
|
worker_machine_type='{{profile.machine}}',
|
|
worker_disk_type=WorkerDiskType('HDD')
|
|
),
|
|
extra_args={
|
|
"environment": '{{profile.environment}}',
|
|
"date": Quote('{{profile.date}}'),
|
|
},
|
|
deployment_config=BatchDeploymentConfig(
|
|
role='{{profile.user_name}}',
|
|
build_target='src/scala/com/twitter/simclusters_v2/scio/multi_type_graph/assemble_multi_type_graph:assemble-multi-type-graph-scio-adhoc-app',
|
|
gcp_deployment_credentials='/var/lib/tss/keys/{{profile.user_name}}/cloud/gcp/dp/shadow.json',
|
|
statebird_config=StatebirdConfig(
|
|
batch_width='PT1H',
|
|
first_time='{{profile.date}}'
|
|
)
|
|
)
|
|
)
|
|
|
|
jobs=[job]
|