Patchwork D6927: ci: report cost to run each job

login
register
mail settings
Submitter phabricator
Date Oct. 1, 2019, 4:41 a.m.
Message ID <822be3c66510e46a3cefc6ac61c7b710@localhost.localdomain>
Download mbox | patch
Permalink /patch/41877/
State Not Applicable
Headers show

Comments

phabricator - Oct. 1, 2019, 4:41 a.m.
indygreg updated this revision to Diff 16724.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D6927?vs=16721&id=16724

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D6927/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D6927

AFFECTED FILES
  contrib/ci/lambda_functions/ci.py
  contrib/ci/lambda_functions/web.py
  contrib/ci/terraform/job_executor.tf

CHANGE DETAILS




To: indygreg, #hg-reviewers
Cc: mercurial-devel

Patch

diff --git a/contrib/ci/terraform/job_executor.tf b/contrib/ci/terraform/job_executor.tf
--- a/contrib/ci/terraform/job_executor.tf
+++ b/contrib/ci/terraform/job_executor.tf
@@ -142,6 +142,7 @@ 
       "ec2:CreateTags",
       "ec2:DescribeInstanceAttribute",
       "ec2:DescribeInstances",
+      "ec2:DescribeSpotInstanceRequests",
     ]
     resources = ["*"]
   }
diff --git a/contrib/ci/lambda_functions/web.py b/contrib/ci/lambda_functions/web.py
--- a/contrib/ci/lambda_functions/web.py
+++ b/contrib/ci/lambda_functions/web.py
@@ -115,6 +115,7 @@ 
                 '<th>Scheduled At</th>',
                 '<th>Start Delay</th>',
                 '<th>Execution Time</th>',
+                '<th>Cost</th>',
                 '<th>Total Tests</th>',
                 '<th>Passed</th>',
                 '<th>Failed</th>',
@@ -136,14 +137,28 @@ 
                         start_time = datetime.datetime.utcfromtimestamp(job_info['start_time'])
                         start_delay = '%ds' % (start_time - schedule_time).total_seconds()
                     else:
+                        start_time = None
                         start_delay = 'n/a'
 
                     if 'end_time' in job_info:
                         end_time = datetime.datetime.utcfromtimestamp(job_info['end_time'])
                         execution_time = '%ds' %  (end_time - start_time).total_seconds()
+
+                        instance_time = (end_time - start_time).total_seconds()
                     else:
                         execution_time = 'n/a'
 
+                        if start_time is not None:
+                            instance_time = (datetime.datetime.utcnow() - start_time).total_seconds()
+                        else:
+                            instance_time = None
+
+                    if 'instance_hourly_cost' in job_info and instance_time is not None:
+                        total_cost = float(job_info['instance_hourly_cost'] )/ 3600.0 * instance_time
+                        total_cost = '$%.3f' % total_cost
+                    else:
+                        total_cost = 'n/a'
+
                     if 'test_count' in job_info:
                         test_count = '%d' % job_info['test_count']
                     else:
@@ -207,6 +222,7 @@ 
                         '<td>%s</td>' % schedule_time.isoformat(),
                         '<td>%s</td>' % start_delay,
                         '<td>%s</td>' % execution_entry,
+                        '<td>%s</td>' % e(total_cost),
                         '<td>%s</td>' % test_count,
                         '<td>%s</td>' % pass_count,
                         '<td>%s</td>' % fail_entry,
diff --git a/contrib/ci/lambda_functions/ci.py b/contrib/ci/lambda_functions/ci.py
--- a/contrib/ci/lambda_functions/ci.py
+++ b/contrib/ci/lambda_functions/ci.py
@@ -116,6 +116,7 @@ 
     state = event['detail']['state']
     print('received %s for %s' % (state, instance_id))
 
+    ec2_client = boto3.client('ec2')
     ec2 = boto3.resource('ec2')
     dynamodb = boto3.resource('dynamodb')
 
@@ -132,7 +133,7 @@ 
 
     job_table = dynamodb.Table(os.environ['DYNAMODB_JOB_TABLE'])
 
-    react_to_instance_state_change(job_table, instance, state)
+    react_to_instance_state_change(ec2_client, job_table, instance, state)
 
 
 def handle_try_server_upload(event, context):
@@ -645,7 +646,7 @@ 
             )
 
 
-def react_to_instance_state_change(job_table, instance, state):
+def react_to_instance_state_change(ec2, job_table, instance, state):
     """React to a CI worker instance state change."""
     now = decimal.Decimal(time.time())
 
@@ -690,17 +691,32 @@ 
     # New instance/job seen. Record that.
     if state == 'pending':
         print('recording running state for job %s' % job_id)
+
+        # Try to record the cost to running this instance.
+        hourly_cost = None
+
+        if instance.spot_instance_request_id:
+            spot_instance_requests = ec2.describe_spot_instance_requests(
+                SpotInstanceRequestIds=[instance.spot_instance_request_id],
+            )['SpotInstanceRequests']
+
+            if spot_instance_requests:
+                hourly_cost = decimal.Decimal(
+                    spot_instance_requests[0]['ActualBlockHourlyPrice'])
+
         job_table.update_item(
             Key={'job_id': job_id},
             UpdateExpression=(
                 'set execution_state = :state, '
                 'instance_id = :instance_id, '
+                'instance_hourly_cost = :hourly_cost, '
                 'start_time = :start_time, '
                 'exit_clean = :exit_clean'
             ),
             ExpressionAttributeValues={
                 ':state': 'running',
                 ':instance_id': instance.instance_id,
+                ':hourly_cost': hourly_cost,
                 ':start_time': now,
                 ':exit_clean': False,
             },