# -*- coding: utf-8 -*- # Copyright 2015 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Module to download and run the CIPD client. CIPD is the Chrome Infra Package Deployer, a simple method of resolving a package/version into a GStorage link and installing them. """ from __future__ import print_function import hashlib import json import os import pprint import tempfile import httplib2 from six.moves import urllib import autotest_lib.utils.frozen_chromite.lib.cros_logging as log from autotest_lib.utils.frozen_chromite.lib import cache from autotest_lib.utils.frozen_chromite.lib import osutils from autotest_lib.utils.frozen_chromite.lib import path_util from autotest_lib.utils.frozen_chromite.lib import cros_build_lib from autotest_lib.utils.frozen_chromite.utils import memoize # pylint: disable=line-too-long # CIPD client to download. # # This is version "git_revision:db7a486094873e3944b8e27ab5b23a3ae3c401e7". # # To switch to another version: # 1. Find it in CIPD Web UI, e.g. # https://chrome-infra-packages.appspot.com/p/infra/tools/cipd/linux-amd64/+/latest # 2. Look up SHA256 there. # pylint: enable=line-too-long CIPD_CLIENT_PACKAGE = 'infra/tools/cipd/linux-amd64' CIPD_CLIENT_SHA256 = ( 'ea6b7547ddd316f32fd9974f598949c3f8f22f6beb8c260370242d0d84825162') CHROME_INFRA_PACKAGES_API_BASE = ( 'https://chrome-infra-packages.appspot.com/prpc/cipd.Repository/') class Error(Exception): """Raised on fatal errors.""" def _ChromeInfraRequest(method, request): """Makes a request to the Chrome Infra Packages API with httplib2. Args: method: Name of RPC method to call. request: RPC request body. Returns: Deserialized RPC response body. """ resp, body = httplib2.Http().request( uri=CHROME_INFRA_PACKAGES_API_BASE+method, method='POST', headers={ 'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'chromite', }, body=json.dumps(request)) if resp.status != 200: raise Error('Got HTTP %d from CIPD %r: %s' % (resp.status, method, body)) try: return json.loads(body.lstrip(b")]}'\n")) except ValueError: raise Error('Bad response from CIPD server:\n%s' % (body,)) def _DownloadCIPD(instance_sha256): """Finds the CIPD download link and requests the binary. Args: instance_sha256: The version of CIPD client to download. Returns: The CIPD binary as a string. """ # Grab the signed URL to fetch the client binary from. resp = _ChromeInfraRequest('DescribeClient', { 'package': CIPD_CLIENT_PACKAGE, 'instance': { 'hashAlgo': 'SHA256', 'hexDigest': instance_sha256, }, }) if 'clientBinary' not in resp: log.error( 'Error requesting the link to download CIPD from. Got:\n%s', pprint.pformat(resp)) raise Error('Failed to bootstrap CIPD client') # Download the actual binary. http = httplib2.Http(cache=None) response, binary = http.request(uri=resp['clientBinary']['signedUrl']) if response.status != 200: raise Error('Got a %d response from Google Storage.' % response.status) # Check SHA256 matches what server expects. digest = hashlib.sha256(binary).hexdigest() for alias in resp['clientRefAliases']: if alias['hashAlgo'] == 'SHA256': if digest != alias['hexDigest']: raise Error( 'Unexpected CIPD client SHA256: got %s, want %s' % (digest, alias['hexDigest'])) break else: raise Error("CIPD server didn't provide expected SHA256") return binary class CipdCache(cache.RemoteCache): """Supports caching of the CIPD download.""" def _Fetch(self, url, local_path): instance_sha256 = urllib.parse.urlparse(url).netloc binary = _DownloadCIPD(instance_sha256) log.info('Fetched CIPD package %s:%s', CIPD_CLIENT_PACKAGE, instance_sha256) osutils.WriteFile(local_path, binary, mode='wb') os.chmod(local_path, 0o755) def GetCIPDFromCache(): """Checks the cache, downloading CIPD if it is missing. Returns: Path to the CIPD binary. """ cache_dir = os.path.join(path_util.GetCacheDir(), 'cipd') bin_cache = CipdCache(cache_dir) key = (CIPD_CLIENT_SHA256,) ref = bin_cache.Lookup(key) ref.SetDefault('cipd://' + CIPD_CLIENT_SHA256) return ref.path def GetInstanceID(cipd_path, package, version, service_account_json=None): """Get the latest instance ID for ref latest. Args: cipd_path: The path to a cipd executable. GetCIPDFromCache can give this. package: A string package name. version: A string version of package. service_account_json: The path of the service account credentials. Returns: A string instance ID. """ service_account_flag = [] if service_account_json: service_account_flag = ['-service-account-json', service_account_json] result = cros_build_lib.run( [cipd_path, 'resolve', package, '-version', version] + service_account_flag, capture_output=True, encoding='utf-8') # An example output of resolve is like: # Packages:\n package:instance_id return result.output.splitlines()[-1].split(':')[-1] @memoize.Memoize def InstallPackage(cipd_path, package, instance_id, destination, service_account_json=None): """Installs a package at a given destination using cipd. Args: cipd_path: The path to a cipd executable. GetCIPDFromCache can give this. package: A package name. instance_id: The version of the package to install. destination: The folder to install the package under. service_account_json: The path of the service account credentials. Returns: The path of the package. """ destination = os.path.join(destination, package) service_account_flag = [] if service_account_json: service_account_flag = ['-service-account-json', service_account_json] with tempfile.NamedTemporaryFile() as f: f.write(('%s %s' % (package, instance_id)).encode('utf-8')) f.flush() cros_build_lib.run( [cipd_path, 'ensure', '-root', destination, '-list', f.name] + service_account_flag, capture_output=True) return destination def CreatePackage(cipd_path, package, in_dir, tags, refs, cred_path=None): """Create (build and register) a package using cipd. Args: cipd_path: The path to a cipd executable. GetCIPDFromCache can give this. package: A package name. in_dir: The directory to create the package from. tags: A mapping of tags to apply to the package. refs: An Iterable of refs to apply to the package. cred_path: The path of the service account credentials. """ args = [ cipd_path, 'create', '-name', package, '-in', in_dir, ] for key, value in tags.items(): args.extend(['-tag', '%s:%s' % (key, value)]) for ref in refs: args.extend(['-ref', ref]) if cred_path: args.extend(['-service-account-json', cred_path]) cros_build_lib.run(args, capture_output=True) def BuildPackage(cipd_path, package, in_dir, outfile): """Build a package using cipd. Args: cipd_path: The path to a cipd executable. GetCIPDFromCache can give this. package: A package name. in_dir: The directory to create the package from. outfile: Output file. Should have extension .cipd """ args = [ cipd_path, 'pkg-build', '-name', package, '-in', in_dir, '-out', outfile, ] cros_build_lib.run(args, capture_output=True) def RegisterPackage(cipd_path, package_file, tags, refs, cred_path=None): """Register and upload a package using cipd. Args: cipd_path: The path to a cipd executable. GetCIPDFromCache can give this. package_file: The path to a .cipd package file. tags: A mapping of tags to apply to the package. refs: An Iterable of refs to apply to the package. cred_path: The path of the service account credentials. """ args = [cipd_path, 'pkg-register', package_file] for key, value in tags.items(): args.extend(['-tag', '%s:%s' % (key, value)]) for ref in refs: args.extend(['-ref', ref]) if cred_path: args.extend(['-service-account-json', cred_path]) cros_build_lib.run(args, capture_output=True)