mirror of
https://github.com/twitter/the-algorithm.git
synced 2025-06-12 04:49:42 +02:00
Open-sourcing Unified User Actions
Unified User Action (UUA) is a centralized, real-time stream of user actions on Twitter, consumed by various product, ML, and marketing teams. UUA makes sure all internal teams consume the uniformed user actions data in an accurate and fast way.
This commit is contained in:
twitter-team
@ -0,0 +1,46 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-kill-staging-services",
|
||||
"config-files": [],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 1"
|
||||
},
|
||||
"dependencies": [],
|
||||
"steps": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "script",
|
||||
"name": "uua-kill-staging-services",
|
||||
"keytab": "/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab",
|
||||
"repository": "source",
|
||||
"command": "bash unified_user_actions/scripts/kill_staging.sh",
|
||||
"dependencies": [{
|
||||
"version": "latest",
|
||||
"role": "aurora",
|
||||
"name": "aurora"
|
||||
}],
|
||||
"timeout": "10.minutes"
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "unified_user_actions_dev"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "unified_user_actions_dev"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-iesource-prod",
|
||||
"config-files": [
|
||||
"rekey-uua-iesource.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-iesource",
|
||||
"artifact": "./dist/rekey-uua-iesource.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-iesource-prod-atla",
|
||||
"key": "atla/discode/prod/rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"name": "rekey-uua-iesource-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/rekey-uua-iesource"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-iesource-staging",
|
||||
"config-files": [
|
||||
"rekey-uua-iesource.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-iesource-staging",
|
||||
"artifact": "./dist/rekey-uua-iesource.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-iesource-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/rekey-uua-iesource"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
204
unified_user_actions/service/deploy/rekey-uua-iesource.aurora
Normal file
204
unified_user_actions/service/deploy/rekey-uua-iesource.aurora
Normal file
@ -0,0 +1,204 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'rekey-uua-iesource'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 250)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/cdm-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'interaction_events')
|
||||
sink_topics = Default(String, 'uua_keyed')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}-{{cluster}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 500,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
|
||||
### pdxa right now doesn't have InteractionEvents topic
|
||||
PRODUCTION_PDXA = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls'
|
||||
)
|
||||
|
||||
STAGING_PDXA = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL_PDXA = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls'
|
||||
)
|
||||
|
||||
prod_job_pdxa = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION_PDXA)
|
||||
|
||||
staging_job_pdxa = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING_PDXA)
|
||||
|
||||
devel_job_pdxa = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL_PDXA)
|
||||
|
||||
jobs.append(prod_job_pdxa(cluster = 'pdxa'))
|
||||
jobs.append(staging_job_pdxa(cluster = 'pdxa'))
|
||||
jobs.append(devel_job_pdxa(cluster = 'pdxa'))
|
||||
66
unified_user_actions/service/deploy/rekey-uua-prod.workflow
Normal file
66
unified_user_actions/service/deploy/rekey-uua-prod.workflow
Normal file
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-prod",
|
||||
"config-files": [
|
||||
"rekey-uua.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua",
|
||||
"artifact": "./dist/rekey-uua.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-prod-atla",
|
||||
"key": "atla/discode/prod/rekey-uua"
|
||||
},
|
||||
{
|
||||
"name": "rekey-uua-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/rekey-uua"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-staging",
|
||||
"config-files": [
|
||||
"rekey-uua.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-staging",
|
||||
"artifact": "./dist/rekey-uua.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/rekey-uua"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/rekey-uua.aurora
Normal file
167
unified_user_actions/service/deploy/rekey-uua.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'rekey-uua'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 100)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'unified_user_actions')
|
||||
sink_topics = Default(String, 'uua_keyed')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 100,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-ads-callback-engagements-prod",
|
||||
"config-files": [
|
||||
"uua-ads-callback-engagements.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-ads-callback-engagements",
|
||||
"artifact": "./dist/uua-ads-callback-engagements.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-prod-atla",
|
||||
"key": "atla/discode/prod/uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-ads-callback-engagements"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-ads-callback-engagements-staging",
|
||||
"config-files": [
|
||||
"uua-ads-callback-engagements.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-ads-callback-engagements-staging",
|
||||
"artifact": "./dist/uua-ads-callback-engagements.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-ads-callback-engagements"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-ads-callback-engagements'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 50)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/ads-callback-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'ads_spend_prod')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-client-event-prod",
|
||||
"config-files": [
|
||||
"uua-client-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-client-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-client-event",
|
||||
"artifact": "./dist/uua-client-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-client-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-client-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-client-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-client-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-client-event-staging",
|
||||
"config-files": [
|
||||
"uua-client-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-client-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-client-event-staging",
|
||||
"artifact": "./dist/uua-client-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-client-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-client-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
174
unified_user_actions/service/deploy/uua-client-event.aurora
Normal file
174
unified_user_actions/service/deploy/uua-client-event.aurora
Normal file
@ -0,0 +1,174 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-client-event'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 1000)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/client-events:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'client_event')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:MaxMetaspaceSize=536870912'
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
# CE events is about 0.4-0.6kb per message on the consumer side. A fetch size of 6~18 MB get us
|
||||
# about 10k ~ 20k of messages per batch. This fits the size of our pending requests queue and
|
||||
# within the limit of the max poll records.
|
||||
' -kafka.consumer.fetch.max=9.megabytes'
|
||||
' -kafka.consumer.fetch.min=3.megabytes'
|
||||
' -kafka.max.poll.records=40000'
|
||||
' -kafka.commit.interval=20.seconds'
|
||||
' -kafka.producer.batch.size=4.megabytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=100.millisecond'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=4'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 1000,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-email-notification-event-prod",
|
||||
"config-files": [
|
||||
"uua-email-notification-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-email-notification-event",
|
||||
"artifact": "./dist/uua-email-notification-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-email-notification-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-email-notification-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-email-notification-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-email-notification-event-staging",
|
||||
"config-files": [
|
||||
"uua-email-notification-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-email-notification-event-staging",
|
||||
"artifact": "./dist/uua-email-notification-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-email-notification-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-email-notification-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,169 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-email-notification-event'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'notifications')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.max.poll.records=20000'
|
||||
' -kafka.commit.interval=10.seconds'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-enricher-staging",
|
||||
"config-files": [
|
||||
"uua-enricher.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-enricher"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-enricher-staging",
|
||||
"artifact": "./dist/uua-enricher.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-enricher-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-enricher"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
151
unified_user_actions/service/deploy/uua-enricher.aurora
Normal file
151
unified_user_actions/service/deploy/uua-enricher.aurora
Normal file
@ -0,0 +1,151 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-enricher'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 6 * GB
|
||||
RAM_SIZE = 8 * GB
|
||||
DISK_SIZE = 3 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.application.id={{name}}.{{environment}}'
|
||||
' -kafka.application.num.instances={{instances}}' # Used for static partitioning
|
||||
' -kafka.application.server={{mesos.instance}}.{{name}}.{{environment}}.{{role}}.service.{{cluster}}.twitter.com:80'
|
||||
' -com.twitter.finatra.kafkastreams.config.principal={{role}}'
|
||||
' -thrift.client.id={{name}}.{{environment}}'
|
||||
' -service.identifier="{{role}}:{{name}}:{{environment}}:{{cluster}}"'
|
||||
' -local.cache.ttl.seconds=86400'
|
||||
' -local.cache.max.size=400000000'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers = '/s/kafka/custdevel:kafka-tls'
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-enrichment-planner-staging",
|
||||
"config-files": [
|
||||
"uua-enrichment-planner.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-enrichment-planner"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-enrichment-planner-staging",
|
||||
"artifact": "./dist/uua-enrichment-planner.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-enricher-enrichment-planner-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-enrichment-planner"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,156 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-enrichment-planner'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 50)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_output_server = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version(default_version='live')
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem'
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.output.server={{profile.kafka_output_server}}'
|
||||
' -kafka.application.id=uua-enrichment-planner'
|
||||
' -com.twitter.finatra.kafkastreams.config.principal={{role}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_output_server = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
instances = 2,
|
||||
kafka_output_server = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-favorite-archival-events-prod",
|
||||
"config-files": [
|
||||
"uua-favorite-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-favorite-archival-events",
|
||||
"artifact": "./dist/uua-favorite-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-favorite-archival-events-prod-atla",
|
||||
"key": "atla/discode/prod/uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"name": "uua-favorite-archival-events-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-favorite-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-favorite-archival-events-staging",
|
||||
"config-files": [
|
||||
"uua-favorite-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-favorite-archival-events-staging",
|
||||
"artifact": "./dist/uua-favorite-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-favorite-archival-events-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-favorite-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-favorite-archival-events'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'favorite_archival_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-retweet-archival-events-prod",
|
||||
"config-files": [
|
||||
"uua-retweet-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-retweet-archival-events",
|
||||
"artifact": "./dist/uua-retweet-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-retweet-archival-events-prod-atla",
|
||||
"key": "atla/discode/prod/uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"name": "uua-retweet-archival-events-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-retweet-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-retweet-archival-events-staging",
|
||||
"config-files": [
|
||||
"uua-retweet-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-retweet-archival-events-staging",
|
||||
"artifact": "./dist/uua-retweet-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-retweet-archival-events-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-retweet-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-retweet-archival-events'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'retweet_archival_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-social-graph-prod",
|
||||
"config-files": [
|
||||
"uua-social-graph.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-social-graph"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-social-graph",
|
||||
"artifact": "./dist/uua-social-graph.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-social-graph-prod-atla",
|
||||
"key": "atla/discode/prod/uua-social-graph"
|
||||
},
|
||||
{
|
||||
"name": "uua-social-graph-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-social-graph"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-social-graph-staging",
|
||||
"config-files": [
|
||||
"uua-social-graph.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-social-graph"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-social-graph-staging",
|
||||
"artifact": "./dist/uua-social-graph.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-social-graph-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-social-graph"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-social-graph.aurora
Normal file
167
unified_user_actions/service/deploy/uua-social-graph.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-social-graph'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'social_write_event')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.second'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tls-favs-prod",
|
||||
"config-files": [
|
||||
"uua-tls-favs.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tls-favs",
|
||||
"artifact": "./dist/uua-tls-favs.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tls-favs-prod-atla",
|
||||
"key": "atla/discode/prod/uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"name": "uua-tls-favs-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-tls-favs"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tls-favs-staging",
|
||||
"config-files": [
|
||||
"uua-tls-favs.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tls-favs-staging",
|
||||
"artifact": "./dist/uua-tls-favs.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tls-favs-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-tls-favs"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-tls-favs.aurora
Normal file
167
unified_user_actions/service/deploy/uua-tls-favs.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-tls-favs'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'timeline_service_favorites')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tweetypie-event-prod",
|
||||
"config-files": [
|
||||
"uua-tweetypie-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tweetypie-event",
|
||||
"artifact": "./dist/uua-tweetypie-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tweetypie-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-tweetypie-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-tweetypie-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tweetypie-event-staging",
|
||||
"config-files": [
|
||||
"uua-tweetypie-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tweetypie-event-staging",
|
||||
"artifact": "./dist/uua-tweetypie-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tweetypie-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-tweetypie-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-tweetypie-event.aurora
Normal file
167
unified_user_actions/service/deploy/uua-tweetypie-event.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-tweetypie-event'
|
||||
|
||||
CPU_NUM = 2
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/tweet-events:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'tweet_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-user-modification-prod",
|
||||
"config-files": [
|
||||
"uua-user-modification.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-user-modification"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-user-modification",
|
||||
"artifact": "./dist/uua-user-modification.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-user-modification-prod-atla",
|
||||
"key": "atla/discode/prod/uua-user-modification"
|
||||
},
|
||||
{
|
||||
"name": "uua-user-modification-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-user-modification"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-user-modification-staging",
|
||||
"config-files": [
|
||||
"uua-user-modification.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-user-modification"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-user-modification-staging",
|
||||
"artifact": "./dist/uua-user-modification.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-user-modification-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-user-modification"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-user-modification.aurora
Normal file
167
unified_user_actions/service/deploy/uua-user-modification.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-user-modification'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'user_modifications')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
Reference in New Issue
Block a user