diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | LICENSE | 1 | ||||
| -rw-r--r-- | README.md | 11 | ||||
| -rwxr-xr-x | arch-image.py | 11 | ||||
| -rwxr-xr-x | arch-staging.py | 147 | ||||
| -rwxr-xr-x | build-arch-on-gce.sh | 4 | ||||
| -rwxr-xr-x | build-gce-arch.py | 316 | ||||
| -rwxr-xr-x | gcevm-script-build-arch.sh (renamed from build-arch-on-gce-remote.sh) | 37 | ||||
| -rw-r--r-- | utils.py | 55 |
9 files changed, 277 insertions, 307 deletions
@@ -4,3 +4,5 @@ *.pyc *.raw __pycache__/* +.c9/* + @@ -200,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -
\ No newline at end of file @@ -6,6 +6,9 @@ The image is configured close to the recommendations listed on [Building an imag These scripts are written in Python3. +## Premade Image +arch-v20150903 - [gs://jeremyje/archlinux-images/arch-v20150903.tar.gz](https://storage.googleapis.com/jeremyje/archlinux-images/arch-v20150903.tar.gz) + ## Usage ### Install and Configure Cloud SDK (one time setup) @@ -35,10 +38,6 @@ gsutil mb gs://${BUCKET} # Install Required Packages # Arch Linux sudo pacman -S python haveged -# Debian -sudo apt-get -y install python3 haveged -# Redhat -sudo yum install -y python3 haveged ./build-gce-arch.py --verbose # Upload to Cloud Storage @@ -58,3 +57,7 @@ gcloud compute images insert archlinux \ ## Licensing All files in this repository are under the [Apache License, Version 2.0](LICENSE) unless noted otherwise. + + +## Disclaimer +Google Inc. does not provide any support or guarantees for this project or the images provided. diff --git a/arch-image.py b/arch-image.py index 9401412..e6cbb04 100755 --- a/arch-image.py +++ b/arch-image.py @@ -261,6 +261,7 @@ def main(): ConfigMessageOfTheDay() ConfigureSecurity() ConfigureSerialPortOutput() + DisableUnusedServices() OptimizePackages() @@ -284,7 +285,7 @@ def OptimizePackages(): def SetupLocale(): - utils.LogStep('Set Locale to US English (UTF-8)') + utils.LogStep('Set Locale to US English (UTF-8)') utils.SetupArchLocale() @@ -307,9 +308,6 @@ def ConfigureKernel(): def InstallBootloader(device, uuid, debugmode): utils.LogStep('Install Syslinux bootloader') - ''' - utils.Run(['syslinux-install_update', '-i', '-a', '-m']) - ''' utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) utils.CreateDirectory('/boot/syslinux') utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') @@ -352,7 +350,10 @@ def InstallBootloader(device, uuid, debugmode): 'APPEND root=', boot_spec) - +def DisableUnusedServices(): + utils.DisableService('getty@tty1.service') + utils.DisableService('graphical.target') + def ForwardSystemdToConsole(): utils.LogStep('Installing syslinux bootloader') utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes') diff --git a/arch-staging.py b/arch-staging.py deleted file mode 100755 index 344e7b7..0000000 --- a/arch-staging.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging -import os -import sys - -import utils - -COMPUTE_IMAGE_PACKAGES_GIT_URL = ( - 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') -IMAGE_FILE='disk.raw' -SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() -SETUP_PACKAGES = 'pacman wget gcc make parted git setconf libaio sudo'.split() -IMAGE_PACKAGES = ('base tar wget ' - 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' - 'ntp psmisc openssh udev less bash-completion zip unzip ' - 'python2 python3').split() - - -def main(): - args = utils.DecodeArgs(sys.argv[1]) - utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) - logging.info('Setup Bootstrapper Environment') - utils.SetupArchLocale() - InstallPackagesForStagingEnvironment() - image_path = os.path.join(os.getcwd(), IMAGE_FILE) - CreateImage(image_path, size_gb=int(args['size_gb'])) - mount_path = utils.CreateTempDirectory(base_dir='/') - image_mapping = utils.ImageMapper(image_path, mount_path) - try: - image_mapping.Map() - primary_mapping = image_mapping.GetFirstMapping() - image_mapping_path = primary_mapping['path'] - FormatImage(image_mapping_path) - try: - image_mapping.Mount() - utils.CreateDirectory('/run/shm') - utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) - InstallArchLinux(mount_path) - disk_uuid = SetupFileSystem(mount_path, image_mapping_path) - ConfigureArchInstall( - args, mount_path, primary_mapping['parent'], disk_uuid) - utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) - PurgeDisk(mount_path) - finally: - image_mapping.Unmount() - ShrinkDisk(image_mapping_path) - finally: - image_mapping.Unmap() - utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) - utils.Sync() - - -def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): - relative_builder_path = utils.CopyBuilder(mount_path) - utils.LogStep('Download compute-image-packages') - packages_dir = utils.CreateTempDirectory(mount_path) - utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) - utils.CreateDirectory(os.path.join(mount_path, '')) - packages_dir = os.path.relpath(packages_dir, mount_path) - params = { - 'packages_dir': '/%s' % packages_dir, - 'device': parent_path, - 'disk_uuid': disk_uuid, - 'accounts': args['accounts'], - 'debugmode': args['debugmode'], - } - params.update(args) - config_arch_py = os.path.join( - '/', relative_builder_path, 'arch-image.py') - utils.RunChroot(mount_path, - '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), - use_custom_path=False) - utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) - - -def InstallPackagesForStagingEnvironment(): - utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) - utils.InstallPackages(SETUP_PACKAGES) - utils.SetupArchLocale() - utils.AurInstall(name='multipath-tools-git') - utils.AurInstall(name='zerofree') - - -def CreateImage(image_path, size_gb=10, fs_type='ext4'): - utils.LogStep('Create Image') - utils.Run(['rm', '-f', image_path]) - utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) - utils.Run(['parted', image_path, 'mklabel', 'msdos']) - utils.Run(['parted', image_path, 'mkpart', 'primary', - fs_type, '1', str(int(size_gb) * 1024)]) - - -def FormatImage(image_mapping_path): - utils.LogStep('Format Image') - utils.Run(['mkfs', image_mapping_path]) - utils.Sync() - - -def InstallArchLinux(base_dir): - utils.LogStep('Install Arch Linux') - utils.Pacstrap(base_dir, IMAGE_PACKAGES) - - -def SetupFileSystem(base_dir, image_mapping_path): - utils.LogStep('File Systems') - _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], - capture_output=True) - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) - _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', - '-o', 'value', - image_mapping_path], - capture_output=True) - disk_uuid = disk_uuid.strip() - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), - 'UUID=%s / ext4 defaults 0 1' % disk_uuid) - utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) - return disk_uuid - - -def PurgeDisk(mount_path): - paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] - for path in paths: - utils.DeleteDirectory(os.path.join(mount_path, path)) - - -def ShrinkDisk(image_mapping_path): - utils.LogStep('Shrink Disk') - utils.Run(['zerofree', image_mapping_path]) - - -main() diff --git a/build-arch-on-gce.sh b/build-arch-on-gce.sh index d1122e2..3f136b0 100755 --- a/build-arch-on-gce.sh +++ b/build-arch-on-gce.sh @@ -46,10 +46,10 @@ function PrintHelp() { function DeployVm() { echo "Creating Instance, ${INSTANCE_NAME}" gcloud compute instances create ${INSTANCE_NAME} \ - --image debian-7-backports \ + --image ubuntu-14-10 \ --machine-type ${MACHINE_TYPE} \ --zone ${ZONE_NAME} \ - --metadata-from-file startup-script=build-arch-on-gce-remote.sh \ + --metadata-from-file startup-script=gcevm-script-build-arch.sh \ --metadata \ script-params="${SCRIPT_PARAMS}" \ instance-name="${INSTANCE_NAME}" \ diff --git a/build-gce-arch.py b/build-gce-arch.py index ee487b6..f64dfb3 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -15,43 +15,40 @@ # limitations under the License. -"""Arch Linux Image Builder for GCE. - -This script creates a clean Arch Linux image that can be used in Google Compute -Engine. - -Usage: ./build-gce-arch.py -> archlinux-***.tar.gz - ./build-gce-arch.py --packages docker - ./build-gce-arch.py --help -> Detailed help. -""" import argparse import os import logging +import sys from datetime import date import utils - -DEFAULT_MIRROR = 'http://mirrors.kernel.org/archlinux/$repo/os/$arch' -#DEFAULT_MIRROR = 'http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch' -TARGET_ARCH = 'x86_64' +COMPUTE_IMAGE_PACKAGES_GIT_URL = ( + 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') +IMAGE_FILE='disk.raw' +SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() +SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' + 'fakeroot arch-install-scripts').split() +IMAGE_PACKAGES = ('base tar wget ' + 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' + 'ntp psmisc openssh udev less bash-completion zip unzip ' + 'python2 python3').split() def main(): args = ParseArgs() utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) + logging.info('Arch Linux Image Builder') + logging.info('========================') + workspace_dir = None image_file = None try: - workspace_dir = utils.CreateTempDirectory() - bootstrap_file = DownloadArchBootstrap(args.bootstrap) - utils.Untar(bootstrap_file, workspace_dir) - arch_root = PrepareBootstrap(workspace_dir, args.mirror, not args.nopacmankeys) - relative_builder_path = utils.CopyBuilder(arch_root) - ChrootIntoArchAndBuild(arch_root, relative_builder_path, args) + InstallPackagesForStagingEnvironment() + image_path = CreateArchImage(args) image_name, image_filename, image_description = GetImageNameAndDescription( args.outfile) - image_file = SaveImage(arch_root, image_filename) + image_file = SaveImage(image_path, image_filename) if args.upload and image_file: UploadImage(image_file, args.upload, make_public=args.public) if args.register: @@ -61,7 +58,165 @@ def main(): if not args.nocleanup and workspace_dir: utils.DeleteDirectory(workspace_dir) - + +def CreateArchImage(args): + image_path = os.path.join(os.getcwd(), IMAGE_FILE) + CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) + mount_path = utils.CreateTempDirectory(base_dir='/') + image_mapping = utils.ImageMapper(image_path, mount_path) + try: + image_mapping.Map() + primary_mapping = image_mapping.GetFirstMapping() + image_mapping_path = primary_mapping['path'] + FormatImage(image_mapping_path) + try: + image_mapping.Mount() + utils.CreateDirectory('/run/shm') + utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) + InstallArchLinux(mount_path) + disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) + ConfigureArchInstall( + args, mount_path, primary_mapping['parent'], disk_uuid) + utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) + PurgeDisk(mount_path) + finally: + image_mapping.Unmount() + ShrinkDisk(image_mapping_path) + finally: + image_mapping.Unmap() + utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) + utils.Sync() + return image_path + + +def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): + relative_builder_path = utils.CopyBuilder(mount_path) + utils.LogStep('Download compute-image-packages') + packages_dir = utils.CreateTempDirectory(mount_path) + utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) + utils.CreateDirectory(os.path.join(mount_path, '')) + packages_dir = os.path.relpath(packages_dir, mount_path) + params = { + 'packages_dir': '/%s' % packages_dir, + 'device': parent_path, + 'disk_uuid': disk_uuid, + 'accounts': args.accounts, + 'debugmode': args.debug, + 'quiet': args.quiet, + 'verbose': args.verbose, + 'packages': args.packages, + 'size_gb': args.size_gb + } + config_arch_py = os.path.join( + '/', relative_builder_path, 'arch-image.py') + utils.RunChroot(mount_path, + '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), + use_custom_path=False) + utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) + + +def InstallPackagesForStagingEnvironment(): + utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) + utils.InstallPackages(SETUP_PACKAGES) + utils.RemoveBuildUser() + utils.AurInstall(name='multipath-tools-git') + utils.AurInstall(name='zerofree') + + +def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): + utils.LogStep('Create Image') + utils.Run(['rm', '-f', image_path]) + utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) + utils.Run(['parted', image_path, 'mklabel', 'msdos']) + utils.Run(['parted', image_path, 'mkpart', 'primary', + fs_type, '1', str(int(size_gb) * 1024)]) + + +def FormatImage(image_mapping_path): + utils.LogStep('Format Image') + utils.Run(['mkfs', image_mapping_path]) + utils.Sync() + + +def InstallArchLinux(base_dir): + utils.LogStep('Install Arch Linux') + utils.Pacstrap(base_dir, IMAGE_PACKAGES) + + +def SetupFileSystem(base_dir, image_mapping_path, fs_type): + utils.LogStep('File Systems') + _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], + capture_output=True) + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) + _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', + '-o', 'value', + image_mapping_path], + capture_output=True) + disk_uuid = disk_uuid.strip() + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), + 'UUID=%s / %s defaults 0 1' % (disk_uuid, fs_type)) + utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) + return disk_uuid + + +def PurgeDisk(mount_path): + paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] + for path in paths: + utils.DeleteDirectory(os.path.join(mount_path, path)) + + +def ShrinkDisk(image_mapping_path): + utils.LogStep('Shrink Disk') + utils.Run(['zerofree', image_mapping_path]) + + +def SaveImage(arch_root, image_filename): + utils.LogStep('Save Arch Linux Image in GCE format') + source_image_raw = os.path.join(arch_root, 'disk.raw') + image_raw = os.path.join(os.getcwd(), 'disk.raw') + image_file = os.path.join(os.getcwd(), image_filename) + utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) + utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) + return image_file + + +def UploadImage(image_path, gs_path, make_public=False): + utils.LogStep('Upload Image to Cloud Storage') + utils.SecureDeleteFile('~/.gsutil/*.url') + utils.Run(['gsutil', 'rm', gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + utils.Run(['gsutil', 'cp', image_path, gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + if make_public: + utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + + +def AddImageToComputeEngineProject(image_name, gs_path, description): + utils.LogStep('Add image to project') + utils.Run( + ['gcloud', 'compute', 'images', 'delete', image_name, '-q'], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + utils.Run( + ['gcloud', 'compute', 'images', 'create', image_name, '-q', + '--source-uri', gs_path, + '--description', description], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + + +def GetImageNameAndDescription(outfile_name): + today = date.today() + isodate = today.strftime("%Y-%m-%d") + yyyymmdd = today.strftime("%Y%m%d") + image_name = 'arch-v%s' % yyyymmdd + if outfile_name: + image_filename = outfile_name + else: + image_filename = '%s.tar.gz' % image_name + description = 'Arch Linux x86-64 built on %s' % isodate + return image_name, image_filename, description + + def ParseArgs(): parser = argparse.ArgumentParser( description='Arch Linux Image Builder for Compute Engine') @@ -69,14 +224,6 @@ def ParseArgs(): dest='packages', nargs='+', help='Additional packages to install via Pacman.') - parser.add_argument('--mirror', - dest='mirror', - default=DEFAULT_MIRROR, - help='Mirror to download packages from.') - parser.add_argument('--bootstrap', - dest='bootstrap', - help='Arch Linux Bootstrap tarball. ' - '(default: Download latest version)') parser.add_argument('-v', '--verbose', dest='verbose', default=False, @@ -132,113 +279,12 @@ def ParseArgs(): default=False, help='Disables signature checking for pacman packages.', action='store_true') + parser.add_argument('--fs_type', + dest='fs_type', + default='ext4', + help='Verbose console output.', + action='store_true') return parser.parse_args() -def DownloadArchBootstrap(bootstrap_tarball): - utils.LogStep('Download Arch Linux Bootstrap') - if bootstrap_tarball: - url = bootstrap_tarball - sha1sum = None - else: - url, sha1sum = GetLatestBootstrapUrl() - logging.debug('Downloading %s', url) - local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url)) - if os.path.isfile(local_bootstrap): - logging.debug('Using local file instead.') - if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum: - return local_bootstrap - utils.DownloadFile(url, local_bootstrap) - if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum: - raise ValueError('Bad checksum') - return local_bootstrap - - -def ChrootIntoArchAndBuild(arch_root, relative_builder_path, args): - params = { - 'quiet': args.quiet, - 'verbose': args.verbose, - 'packages': args.packages, - 'mirror': args.mirror, - 'accounts': args.accounts, - 'debugmode': args.debug, - 'size_gb': args.size_gb - } - chroot_archenv_script = os.path.join('/', relative_builder_path, - 'arch-staging.py') - utils.RunChroot(arch_root, - '%s "%s"' % (chroot_archenv_script, utils.EncodeArgs(params))) - logging.debug('Bootstrap Chroot: sudo %s/bin/arch-chroot %s/', - arch_root, arch_root) - - -def SaveImage(arch_root, image_filename): - utils.LogStep('Save Arch Linux Image in GCE format') - source_image_raw = os.path.join(arch_root, 'disk.raw') - image_raw = os.path.join(os.getcwd(), 'disk.raw') - image_file = os.path.join(os.getcwd(), image_filename) - utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) - utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) - return image_file - - -def UploadImage(image_path, gs_path, make_public=False): - utils.LogStep('Upload Image to Cloud Storage') - utils.SecureDeleteFile('~/.gsutil/*.url') - utils.Run(['gsutil', 'rm', gs_path]) - utils.Run(['gsutil', 'cp', image_path, gs_path]) - if make_public: - utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) - - -def AddImageToComputeEngineProject(image_name, gs_path, description): - utils.LogStep('Add image to project') - utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) - utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q', - '--source-uri', gs_path, - '--description', description]) - -def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): - utils.LogStep('Setting up Bootstrap Environment') - arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0]) - mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server) - utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'), - mirrorlist) - utils.CreateDirectory(os.path.join(arch_root, 'run/shm')) - if use_pacman_keys: - utils.RunChroot(arch_root, 'pacman-key --init') - utils.RunChroot(arch_root, 'pacman-key --populate archlinux') - else: - utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') - # Install the most basic utilities for the bootstrapper. - utils.RunChroot(arch_root, - 'pacman --noconfirm -Sy python3') - - return arch_root - - -def GetLatestBootstrapUrl(): - base_url = 'http://mirrors.kernel.org/archlinux/iso/latest/' - sha1sums = utils.HttpGet(base_url + 'sha1sums.txt') - items = sha1sums.splitlines() - for item in items: - if TARGET_ARCH in item and 'bootstrap' in item: - entries = item.split() - return base_url + entries[1], entries[0] - raise RuntimeError('Cannot find Arch bootstrap url') - - -def GetImageNameAndDescription(outfile_name): - today = date.today() - isodate = today.strftime("%Y-%m-%d") - yyyymmdd = today.strftime("%Y%m%d") - image_name = 'arch-v%s' % yyyymmdd - if outfile_name: - image_filename = outfile_name - else: - image_filename = '%s.tar.gz' % image_name - description = 'Arch Linux x86-64 built on %s' % isodate - return image_name, image_filename, description - - main() diff --git a/build-arch-on-gce-remote.sh b/gcevm-script-build-arch.sh index 5cd584c..8ddaf67 100755 --- a/build-arch-on-gce-remote.sh +++ b/gcevm-script-build-arch.sh @@ -37,9 +37,23 @@ mkdir -p ${BUILDER_ROOT} echo "Updating Cloud SDK" yes | gcloud components update -echo "Installing Dependencies" -sudo apt-get update -sudo apt-get install -y -qq python3 haveged git +function InstallDependenciesForDebian { + echo "Installing Dependencies (Debian)" + apt-get update + apt-get install -y -qq python3 haveged git +} + +function InstallDependenciesForRedhat { + echo "Installing Dependencies (Redhat)" + rpm -Uvh http://download.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm + yum install -y python3 haveged git +} +if [ -f /etc/redhat-release ] +then + InstallDependenciesForRedhat +else + InstallDependenciesForDebian +fi echo "Getting source code..." git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} @@ -48,6 +62,21 @@ cd ${BUILDER_ROOT} haveged -w 1024 gsutil rm ${REMOTE_IMAGE} ./build-gce-arch.py ${SCRIPT_PARAMS} -cat /var/log/syslog | grep -o "startupscript.*" > builder.log + +function SaveLogForRedhat { + journalctl > builder.log +} + +function SaveLogForDebian { + cat /var/log/syslog | grep -o "startupscript.*" > builder.log +} + +if [ -f /etc/redhat-release ] +then + SaveLogForRedhat +else + SaveLogForDebian +fi + gsutil cp builder.log ${REMOTE_IMAGE}.log gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} @@ -20,6 +20,7 @@ import hashlib import json import os import logging +import pwd import shutil import subprocess import sys @@ -29,6 +30,7 @@ import urllib.request, urllib.error, urllib.parse APP_NAME = 'archbuilder' +BUILDER_USER = APP_NAME ETC_LOCALE_GEN = ''' en_US.UTF-8 UTF-8 en_US ISO-8859-1 @@ -70,9 +72,30 @@ def SudoRun(params, cwd=None, capture_output=False): return Run(params, capture_output=capture_output) +def UserExists(username): + try: + pwd.getpwnam(username) + return True + except: + return False + + +def CreateBuildUser(user=BUILDER_USER): + if not UserExists(user): + home_dir = '/home/%s' % user + Run(['useradd', user, '-d', home_dir]) + Run(['mkdir', home_dir]) + Run(['chown', '%s:%s' % (user, user), home_dir]) + + +def RemoveBuildUser(): + if UserExists(BUILDER_USER): + Run(['userdel', '-r', BUILDER_USER]) + + def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): try: - logging.debug(params) + logging.debug('Run: %s in %s', params, cwd) if env: new_env = os.environ.copy() new_env.update(env) @@ -103,7 +126,7 @@ def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True def DownloadFile(url, file_path): - Run(['wget', '-O', file_path, url]) + Run(['wget', '-O', file_path, url, '-nv']) def HttpGet(url): @@ -150,8 +173,13 @@ def DeleteFileFunc(file_pattern, delete_func): delete_func(item) +def DirectoryExists(dir_path): + return os.path.exists(dir_path) + + def DeleteDirectory(dir_path): - shutil.rmtree(dir_path) + if DirectoryExists(dir_path): + shutil.rmtree(dir_path) def CreateTempDirectory(base_dir=None): @@ -207,9 +235,9 @@ def DebugBash(): def DebugPrintFile(file_path): - logging.info('==================================================================================') + logging.info('==============================================================') logging.info('File: %s', file_path) - logging.info('==================================================================================') + logging.info('==============================================================') Run(['cat', file_path]) @@ -218,19 +246,29 @@ def Sync(): def EnableService(service_name): - Run(['systemctl', 'enable', service_name]) + Run(['systemctl', 'enable', service_name]) + + +def DisableService(service_name): + Run(['systemctl', 'disable', service_name]) def Symlink(source_file, dest_file): Run(['ln', '-s', source_file, dest_file]) +def ChangeDirectoryOwner(username, directory): + SudoRun(['chown', '-R', username, directory]) + + def AurInstall(name=None, pkbuild_url=None): + CreateBuildUser() if name: - pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower()) + pkbuild_url = 'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=%s' % (name.lower()) workspace_dir = CreateTempDirectory() DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) - Run(['makepkg', '--asroot'], cwd=workspace_dir) + ChangeDirectoryOwner(BUILDER_USER, workspace_dir) + Run(['runuser', '-m', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) tarball = tarball[0] Pacman(['-U', tarball], cwd=workspace_dir) @@ -242,7 +280,6 @@ def Pacstrap(base_dir, params): def Pacman(params, cwd=None): - #, '--debug' Run(['pacman', '--noconfirm'] + params, cwd=cwd) |