From f0f7d6405556862932c5d2fa1f95097200585bdc Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 12 Feb 2015 08:25:53 +0000 Subject: Update scripts to work with new Arch install scripts. --- .gitignore | 2 ++ LICENSE | 1 - arch-image.py | 16 +++++---- arch-staging.py | 3 +- build-arch-on-gce-remote.sh | 53 ----------------------------- build-arch-on-gce.sh | 6 ++-- build-gce-arch.py | 2 +- gcevm-script-build-arch.sh | 82 +++++++++++++++++++++++++++++++++++++++++++++ utils.py | 35 ++++++++++++++++--- 9 files changed, 130 insertions(+), 70 deletions(-) delete mode 100755 build-arch-on-gce-remote.sh create mode 100755 gcevm-script-build-arch.sh diff --git a/.gitignore b/.gitignore index 951b277..02f1795 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ *.pyc *.raw __pycache__/* +.c9/* + diff --git a/LICENSE b/LICENSE index 15f9dd5..a4c5efd 100644 --- a/LICENSE +++ b/LICENSE @@ -200,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - \ No newline at end of file diff --git a/arch-image.py b/arch-image.py index 9401412..0d07044 100755 --- a/arch-image.py +++ b/arch-image.py @@ -261,6 +261,7 @@ def main(): ConfigMessageOfTheDay() ConfigureSecurity() ConfigureSerialPortOutput() + DisableUnusedServices() OptimizePackages() @@ -284,7 +285,7 @@ def OptimizePackages(): def SetupLocale(): - utils.LogStep('Set Locale to US English (UTF-8)') + utils.LogStep('Set Locale to US English (UTF-8)') utils.SetupArchLocale() @@ -307,9 +308,6 @@ def ConfigureKernel(): def InstallBootloader(device, uuid, debugmode): utils.LogStep('Install Syslinux bootloader') - ''' - utils.Run(['syslinux-install_update', '-i', '-a', '-m']) - ''' utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) utils.CreateDirectory('/boot/syslinux') utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') @@ -345,14 +343,20 @@ def InstallBootloader(device, uuid, debugmode): 'systemd.log_target=console', 'systemd.journald.forward_to_syslog=yes', 'systemd.journald.forward_to_kmsg=yes', - 'systemd.journald.forward_to_console=yes',] + 'systemd.journald.forward_to_console=yes', + 'initcall_debug', + 'printk.time=y', + 'init=/usr/lib/systemd/systemd-bootchart',] boot_params = ' '.join(boot_params) boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'APPEND root=', boot_spec) - +def DisableUnusedServices(): + utils.DisableService('getty@tty1.service') + utils.DisableService('graphical.target') + def ForwardSystemdToConsole(): utils.LogStep('Installing syslinux bootloader') utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes') diff --git a/arch-staging.py b/arch-staging.py index 344e7b7..ee659fb 100755 --- a/arch-staging.py +++ b/arch-staging.py @@ -25,7 +25,8 @@ COMPUTE_IMAGE_PACKAGES_GIT_URL = ( 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') IMAGE_FILE='disk.raw' SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() -SETUP_PACKAGES = 'pacman wget gcc make parted git setconf libaio sudo'.split() +SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' + 'fakeroot').split() IMAGE_PACKAGES = ('base tar wget ' 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' 'ntp psmisc openssh udev less bash-completion zip unzip ' diff --git a/build-arch-on-gce-remote.sh b/build-arch-on-gce-remote.sh deleted file mode 100755 index 5cd584c..0000000 --- a/build-arch-on-gce-remote.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build an Arch Linux image from within a GCE Debian VM. - -BUILDER_ROOT=/mnt/archongce/source -INSTANCE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-name) -ZONE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-zone) -SCRIPT_PARAMS=$(/usr/share/google/get_metadata_value attributes/script-params) -SCRIPT_PARAMS="--verbose --register ${SCRIPT_PARAMS}" -GIT_SOURCE_URI=$(/usr/share/google/get_metadata_value attributes/git-source-uri) -REMOTE_IMAGE=$(echo "i = '${SCRIPT_PARAMS}'.split(); print i[i.index('--upload') + 1]" | python) - -echo "Builder Root: ${BUILDER_ROOT}" -echo "Instance Name: ${INSTANCE_NAME}" -echo "Instance Zone: ${ZONE_NAME}" -echo "Source Git Repository: ${GIT_SOURCE_URI}" -echo "Remote Image: ${REMOTE_IMAGE}" -echo "Running script as:" -echo " ./build-gce-arch.py ${SCRIPT_PARAMS}" - -echo "Setup Builder Environment" -mkdir -p ${BUILDER_ROOT} - -echo "Updating Cloud SDK" -yes | gcloud components update - -echo "Installing Dependencies" -sudo apt-get update -sudo apt-get install -y -qq python3 haveged git - -echo "Getting source code..." -git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} - -cd ${BUILDER_ROOT} -haveged -w 1024 -gsutil rm ${REMOTE_IMAGE} -./build-gce-arch.py ${SCRIPT_PARAMS} -cat /var/log/syslog | grep -o "startupscript.*" > builder.log -gsutil cp builder.log ${REMOTE_IMAGE}.log -gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} diff --git a/build-arch-on-gce.sh b/build-arch-on-gce.sh index d1122e2..632586d 100755 --- a/build-arch-on-gce.sh +++ b/build-arch-on-gce.sh @@ -19,7 +19,7 @@ INSTANCE_ID=${RANDOM} INSTANCE_NAME=archbuilder${INSTANCE_ID} ZONE_NAME=us-central1-f MACHINE_TYPE=n1-standard-2 -GIT_SOURCE_URI=https://github.com/GoogleCloudPlatform/compute-archlinux-image-builder.git +GIT_SOURCE_URI=https://github.com/jeremyje/compute-archlinux-image-builder.git SCRIPT_PARAMS="$*" function GcloudNotConfiguredHelp() { @@ -46,10 +46,10 @@ function PrintHelp() { function DeployVm() { echo "Creating Instance, ${INSTANCE_NAME}" gcloud compute instances create ${INSTANCE_NAME} \ - --image debian-7-backports \ + --image ubuntu-14-10 \ --machine-type ${MACHINE_TYPE} \ --zone ${ZONE_NAME} \ - --metadata-from-file startup-script=build-arch-on-gce-remote.sh \ + --metadata-from-file startup-script=gcevm-script-build-arch.sh \ --metadata \ script-params="${SCRIPT_PARAMS}" \ instance-name="${INSTANCE_NAME}" \ diff --git a/build-gce-arch.py b/build-gce-arch.py index ee487b6..f6d31c4 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -212,7 +212,7 @@ def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') # Install the most basic utilities for the bootstrapper. utils.RunChroot(arch_root, - 'pacman --noconfirm -Sy python3') + 'pacman --noconfirm -Sy python3 sed') return arch_root diff --git a/gcevm-script-build-arch.sh b/gcevm-script-build-arch.sh new file mode 100755 index 0000000..8ddaf67 --- /dev/null +++ b/gcevm-script-build-arch.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build an Arch Linux image from within a GCE Debian VM. + +BUILDER_ROOT=/mnt/archongce/source +INSTANCE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-name) +ZONE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-zone) +SCRIPT_PARAMS=$(/usr/share/google/get_metadata_value attributes/script-params) +SCRIPT_PARAMS="--verbose --register ${SCRIPT_PARAMS}" +GIT_SOURCE_URI=$(/usr/share/google/get_metadata_value attributes/git-source-uri) +REMOTE_IMAGE=$(echo "i = '${SCRIPT_PARAMS}'.split(); print i[i.index('--upload') + 1]" | python) + +echo "Builder Root: ${BUILDER_ROOT}" +echo "Instance Name: ${INSTANCE_NAME}" +echo "Instance Zone: ${ZONE_NAME}" +echo "Source Git Repository: ${GIT_SOURCE_URI}" +echo "Remote Image: ${REMOTE_IMAGE}" +echo "Running script as:" +echo " ./build-gce-arch.py ${SCRIPT_PARAMS}" + +echo "Setup Builder Environment" +mkdir -p ${BUILDER_ROOT} + +echo "Updating Cloud SDK" +yes | gcloud components update + +function InstallDependenciesForDebian { + echo "Installing Dependencies (Debian)" + apt-get update + apt-get install -y -qq python3 haveged git +} + +function InstallDependenciesForRedhat { + echo "Installing Dependencies (Redhat)" + rpm -Uvh http://download.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm + yum install -y python3 haveged git +} +if [ -f /etc/redhat-release ] +then + InstallDependenciesForRedhat +else + InstallDependenciesForDebian +fi + +echo "Getting source code..." +git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} + +cd ${BUILDER_ROOT} +haveged -w 1024 +gsutil rm ${REMOTE_IMAGE} +./build-gce-arch.py ${SCRIPT_PARAMS} + +function SaveLogForRedhat { + journalctl > builder.log +} + +function SaveLogForDebian { + cat /var/log/syslog | grep -o "startupscript.*" > builder.log +} + +if [ -f /etc/redhat-release ] +then + SaveLogForRedhat +else + SaveLogForDebian +fi + +gsutil cp builder.log ${REMOTE_IMAGE}.log +gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} diff --git a/utils.py b/utils.py index dec05da..a98c63c 100644 --- a/utils.py +++ b/utils.py @@ -20,6 +20,7 @@ import hashlib import json import os import logging +import pwd import shutil import subprocess import sys @@ -29,6 +30,7 @@ import urllib.request, urllib.error, urllib.parse APP_NAME = 'archbuilder' +BUILDER_USER = APP_NAME ETC_LOCALE_GEN = ''' en_US.UTF-8 UTF-8 en_US ISO-8859-1 @@ -70,6 +72,19 @@ def SudoRun(params, cwd=None, capture_output=False): return Run(params, capture_output=capture_output) +def UserExists(username): + try: + pwd.getpwnam(username) + return True + except: + return False + + +def CreateBuildUser(): + if not UserExists(BUILDER_USER): + Run(['useradd', BUILDER_USER, '-d', '/home/%s' % BUILDER_USER]) + + def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): try: logging.debug(params) @@ -103,7 +118,7 @@ def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True def DownloadFile(url, file_path): - Run(['wget', '-O', file_path, url]) + Run(['wget', '-O', file_path, url, '-nv']) def HttpGet(url): @@ -207,9 +222,9 @@ def DebugBash(): def DebugPrintFile(file_path): - logging.info('==================================================================================') + logging.info('==============================================================') logging.info('File: %s', file_path) - logging.info('==================================================================================') + logging.info('==============================================================') Run(['cat', file_path]) @@ -218,19 +233,29 @@ def Sync(): def EnableService(service_name): - Run(['systemctl', 'enable', service_name]) + Run(['systemctl', 'enable', service_name]) + + +def DisableService(service_name): + Run(['systemctl', 'disable', service_name]) def Symlink(source_file, dest_file): Run(['ln', '-s', source_file, dest_file]) +def ChangeDirectoryOwner(username, directory): + SudoRun(['chown', '-R', username, directory]) + + def AurInstall(name=None, pkbuild_url=None): + CreateBuildUser() if name: pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower()) workspace_dir = CreateTempDirectory() DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) - Run(['makepkg', '--asroot'], cwd=workspace_dir) + ChangeDirectoryOwner(BUILDER_USER, workspace_dir) + Run(['runuser', '-l', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) tarball = tarball[0] Pacman(['-U', tarball], cwd=workspace_dir) -- cgit v1.3 From 04d570bc5718ecbac252227d9f7eb4d8ef9d50d2 Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 12 Feb 2015 09:00:09 +0000 Subject: Remove debugging information --- arch-image.py | 5 +---- build-arch-on-gce.sh | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch-image.py b/arch-image.py index 0d07044..e6cbb04 100755 --- a/arch-image.py +++ b/arch-image.py @@ -343,10 +343,7 @@ def InstallBootloader(device, uuid, debugmode): 'systemd.log_target=console', 'systemd.journald.forward_to_syslog=yes', 'systemd.journald.forward_to_kmsg=yes', - 'systemd.journald.forward_to_console=yes', - 'initcall_debug', - 'printk.time=y', - 'init=/usr/lib/systemd/systemd-bootchart',] + 'systemd.journald.forward_to_console=yes',] boot_params = ' '.join(boot_params) boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) utils.ReplaceLine('/boot/syslinux/syslinux.cfg', diff --git a/build-arch-on-gce.sh b/build-arch-on-gce.sh index 632586d..3f136b0 100755 --- a/build-arch-on-gce.sh +++ b/build-arch-on-gce.sh @@ -19,7 +19,7 @@ INSTANCE_ID=${RANDOM} INSTANCE_NAME=archbuilder${INSTANCE_ID} ZONE_NAME=us-central1-f MACHINE_TYPE=n1-standard-2 -GIT_SOURCE_URI=https://github.com/jeremyje/compute-archlinux-image-builder.git +GIT_SOURCE_URI=https://github.com/GoogleCloudPlatform/compute-archlinux-image-builder.git SCRIPT_PARAMS="$*" function GcloudNotConfiguredHelp() { -- cgit v1.3 From c4f91336a15724300c670e142c8af707ce074bac Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 3 Sep 2015 00:24:05 -0700 Subject: Updated the builder to work with the latest Arch Linux changes. --- README.md | 11 +- arch-staging.py | 166 ++++++++++++++++++++++++++--- build-gce-arch.py | 313 +++++++++++++++++++++++++++++++----------------------- utils.py | 28 +++-- 4 files changed, 355 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index f3f628e..0c92971 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ The image is configured close to the recommendations listed on [Building an imag These scripts are written in Python3. +## Premade Image +arch-v20150903 - [gs://jeremyje/archlinux-images/arch-v20150903.tar.gz](https://storage.googleapis.com/jeremyje/archlinux-images/arch-v20150903.tar.gz) + ## Usage ### Install and Configure Cloud SDK (one time setup) @@ -35,10 +38,6 @@ gsutil mb gs://${BUCKET} # Install Required Packages # Arch Linux sudo pacman -S python haveged -# Debian -sudo apt-get -y install python3 haveged -# Redhat -sudo yum install -y python3 haveged ./build-gce-arch.py --verbose # Upload to Cloud Storage @@ -58,3 +57,7 @@ gcloud compute images insert archlinux \ ## Licensing All files in this repository are under the [Apache License, Version 2.0](LICENSE) unless noted otherwise. + + +## Disclaimer +Google Inc. does not provide any support or guarnatees for this project or the images provided. diff --git a/arch-staging.py b/arch-staging.py index ee659fb..fbbb7a2 100755 --- a/arch-staging.py +++ b/arch-staging.py @@ -15,9 +15,11 @@ # limitations under the License. -import logging +import argparse import os +import logging import sys +from datetime import date import utils @@ -26,7 +28,7 @@ COMPUTE_IMAGE_PACKAGES_GIT_URL = ( IMAGE_FILE='disk.raw' SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' - 'fakeroot').split() + 'fakeroot arch-install-scripts').split() IMAGE_PACKAGES = ('base tar wget ' 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' 'ntp psmisc openssh udev less bash-completion zip unzip ' @@ -34,13 +36,32 @@ IMAGE_PACKAGES = ('base tar wget ' def main(): - args = utils.DecodeArgs(sys.argv[1]) - utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) - logging.info('Setup Bootstrapper Environment') - utils.SetupArchLocale() - InstallPackagesForStagingEnvironment() + args = ParseArgs() + utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) + logging.info('Arch Linux Image Builder') + logging.info('========================') + + workspace_dir = None + image_file = None + try: + InstallPackagesForStagingEnvironment() + image_path = CreateArchImage(args) + image_name, image_filename, image_description = GetImageNameAndDescription( + args.outfile) + image_file = SaveImage(image_path, image_filename) + if args.upload and image_file: + UploadImage(image_file, args.upload, make_public=args.public) + if args.register: + AddImageToComputeEngineProject( + image_name, args.upload, image_description) + finally: + if not args.nocleanup and workspace_dir: + utils.DeleteDirectory(workspace_dir) + + +def CreateArchImage(args): image_path = os.path.join(os.getcwd(), IMAGE_FILE) - CreateImage(image_path, size_gb=int(args['size_gb'])) + CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) mount_path = utils.CreateTempDirectory(base_dir='/') image_mapping = utils.ImageMapper(image_path, mount_path) try: @@ -53,7 +74,7 @@ def main(): utils.CreateDirectory('/run/shm') utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) InstallArchLinux(mount_path) - disk_uuid = SetupFileSystem(mount_path, image_mapping_path) + disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) ConfigureArchInstall( args, mount_path, primary_mapping['parent'], disk_uuid) utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) @@ -65,6 +86,7 @@ def main(): image_mapping.Unmap() utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) utils.Sync() + return image_path def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): @@ -78,10 +100,13 @@ def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): 'packages_dir': '/%s' % packages_dir, 'device': parent_path, 'disk_uuid': disk_uuid, - 'accounts': args['accounts'], - 'debugmode': args['debugmode'], + 'accounts': args.accounts, + 'debugmode': args.debug, + 'quiet': args.quiet, + 'verbose': args.verbose, + 'packages': args.packages, + 'size_gb': args.size_gb } - params.update(args) config_arch_py = os.path.join( '/', relative_builder_path, 'arch-image.py') utils.RunChroot(mount_path, @@ -93,12 +118,12 @@ def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): def InstallPackagesForStagingEnvironment(): utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) utils.InstallPackages(SETUP_PACKAGES) - utils.SetupArchLocale() + utils.RemoveBuildUser() utils.AurInstall(name='multipath-tools-git') utils.AurInstall(name='zerofree') -def CreateImage(image_path, size_gb=10, fs_type='ext4'): +def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): utils.LogStep('Create Image') utils.Run(['rm', '-f', image_path]) utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) @@ -118,7 +143,7 @@ def InstallArchLinux(base_dir): utils.Pacstrap(base_dir, IMAGE_PACKAGES) -def SetupFileSystem(base_dir, image_mapping_path): +def SetupFileSystem(base_dir, image_mapping_path, fs_type): utils.LogStep('File Systems') _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], capture_output=True) @@ -129,7 +154,7 @@ def SetupFileSystem(base_dir, image_mapping_path): capture_output=True) disk_uuid = disk_uuid.strip() utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), - 'UUID=%s / ext4 defaults 0 1' % disk_uuid) + 'UUID=%s / %s defaults 0 1' % (disk_uuid, fs_type)) utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) return disk_uuid @@ -145,4 +170,113 @@ def ShrinkDisk(image_mapping_path): utils.Run(['zerofree', image_mapping_path]) +def SaveImage(arch_root, image_filename): + utils.LogStep('Save Arch Linux Image in GCE format') + source_image_raw = os.path.join(arch_root, 'disk.raw') + image_raw = os.path.join(os.getcwd(), 'disk.raw') + image_file = os.path.join(os.getcwd(), image_filename) + utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) + utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) + return image_file + + +def UploadImage(image_path, gs_path, make_public=False): + utils.LogStep('Upload Image to Cloud Storage') + utils.SecureDeleteFile('~/.gsutil/*.url') + utils.Run(['gsutil', 'rm', gs_path]) + utils.Run(['gsutil', 'cp', image_path, gs_path]) + if make_public: + utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) + + +def AddImageToComputeEngineProject(image_name, gs_path, description): + utils.LogStep('Add image to project') + utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) + utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q', + '--source-uri', gs_path, + '--description', description]) + + +def GetImageNameAndDescription(outfile_name): + today = date.today() + isodate = today.strftime("%Y-%m-%d") + yyyymmdd = today.strftime("%Y%m%d") + image_name = 'arch-v%s' % yyyymmdd + if outfile_name: + image_filename = outfile_name + else: + image_filename = '%s.tar.gz' % image_name + description = 'Arch Linux x86-64 built on %s' % isodate + return image_name, image_filename, description + + +def ParseArgs(): + parser = argparse.ArgumentParser( + description='Arch Linux Image Builder for Compute Engine') + parser.add_argument('-p', '--packages', + dest='packages', + nargs='+', + help='Additional packages to install via Pacman.') + parser.add_argument('-v', '--verbose', + dest='verbose', + default=False, + help='Verbose console output.', + action='store_true') + parser.add_argument('-q', '--quiet', + dest='quiet', + default=False, + help='Suppress all console output.', + action='store_true') + parser.add_argument('--upload', + dest='upload', + default=None, + help='Google Cloud Storage path to upload to.') + parser.add_argument('--size_gb', + dest='size_gb', + default=10, + help='Volume size of image (in GiB).') + parser.add_argument('--accounts', + dest='accounts', + nargs='+', + help='Space delimited list of user accounts to create on ' + 'the image. Format: username:password') + parser.add_argument('--nocleanup', + dest='nocleanup', + default=False, + help='Prevent cleaning up the image build workspace ' + 'after image has been created.', + action='store_true') + parser.add_argument('--outfile', + dest='outfile', + default=None, + help='Name of the output image file.') + parser.add_argument('--debug', + dest='debug', + default=False, + help='Configure the image for debugging.', + action='store_true') + parser.add_argument('--public', + dest='public', + default=False, + help='Make image file uploaded to Cloud Storage ' + 'available for everyone.', + action='store_true') + parser.add_argument('--register', + dest='register', + default=False, + help='Add the image to Compute Engine project. ' + '(Upload to Cloud Storage required.)', + action='store_true') + parser.add_argument('--nopacmankeys', + dest='nopacmankeys', + default=False, + help='Disables signature checking for pacman packages.', + action='store_true') + parser.add_argument('--fs_type', + dest='fs_type', + default='ext4', + help='Verbose console output.', + action='store_true') + return parser.parse_args() + main() diff --git a/build-gce-arch.py b/build-gce-arch.py index f6d31c4..5a3e2b8 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -15,43 +15,40 @@ # limitations under the License. -"""Arch Linux Image Builder for GCE. - -This script creates a clean Arch Linux image that can be used in Google Compute -Engine. - -Usage: ./build-gce-arch.py -> archlinux-***.tar.gz - ./build-gce-arch.py --packages docker - ./build-gce-arch.py --help -> Detailed help. -""" import argparse import os import logging +import sys from datetime import date import utils - -DEFAULT_MIRROR = 'http://mirrors.kernel.org/archlinux/$repo/os/$arch' -#DEFAULT_MIRROR = 'http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch' -TARGET_ARCH = 'x86_64' +COMPUTE_IMAGE_PACKAGES_GIT_URL = ( + 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') +IMAGE_FILE='disk.raw' +SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() +SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' + 'fakeroot arch-install-scripts').split() +IMAGE_PACKAGES = ('base tar wget ' + 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' + 'ntp psmisc openssh udev less bash-completion zip unzip ' + 'python2 python3').split() def main(): args = ParseArgs() utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) + logging.info('Arch Linux Image Builder') + logging.info('========================') + workspace_dir = None image_file = None try: - workspace_dir = utils.CreateTempDirectory() - bootstrap_file = DownloadArchBootstrap(args.bootstrap) - utils.Untar(bootstrap_file, workspace_dir) - arch_root = PrepareBootstrap(workspace_dir, args.mirror, not args.nopacmankeys) - relative_builder_path = utils.CopyBuilder(arch_root) - ChrootIntoArchAndBuild(arch_root, relative_builder_path, args) + InstallPackagesForStagingEnvironment() + image_path = CreateArchImage(args) image_name, image_filename, image_description = GetImageNameAndDescription( args.outfile) - image_file = SaveImage(arch_root, image_filename) + image_file = SaveImage(image_path, image_filename) if args.upload and image_file: UploadImage(image_file, args.upload, make_public=args.public) if args.register: @@ -61,7 +58,162 @@ def main(): if not args.nocleanup and workspace_dir: utils.DeleteDirectory(workspace_dir) - + +def CreateArchImage(args): + image_path = os.path.join(os.getcwd(), IMAGE_FILE) + CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) + mount_path = utils.CreateTempDirectory(base_dir='/') + image_mapping = utils.ImageMapper(image_path, mount_path) + try: + image_mapping.Map() + primary_mapping = image_mapping.GetFirstMapping() + image_mapping_path = primary_mapping['path'] + FormatImage(image_mapping_path) + try: + image_mapping.Mount() + utils.CreateDirectory('/run/shm') + utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) + InstallArchLinux(mount_path) + disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) + ConfigureArchInstall( + args, mount_path, primary_mapping['parent'], disk_uuid) + utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) + PurgeDisk(mount_path) + finally: + image_mapping.Unmount() + ShrinkDisk(image_mapping_path) + finally: + image_mapping.Unmap() + utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) + utils.Sync() + return image_path + + +def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): + relative_builder_path = utils.CopyBuilder(mount_path) + utils.LogStep('Download compute-image-packages') + packages_dir = utils.CreateTempDirectory(mount_path) + utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) + utils.CreateDirectory(os.path.join(mount_path, '')) + packages_dir = os.path.relpath(packages_dir, mount_path) + params = { + 'packages_dir': '/%s' % packages_dir, + 'device': parent_path, + 'disk_uuid': disk_uuid, + 'accounts': args.accounts, + 'debugmode': args.debug, + 'quiet': args.quiet, + 'verbose': args.verbose, + 'packages': args.packages, + 'size_gb': args.size_gb + } + config_arch_py = os.path.join( + '/', relative_builder_path, 'arch-image.py') + utils.RunChroot(mount_path, + '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), + use_custom_path=False) + utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) + + +def InstallPackagesForStagingEnvironment(): + utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) + utils.InstallPackages(SETUP_PACKAGES) + utils.RemoveBuildUser() + utils.AurInstall(name='multipath-tools-git') + utils.AurInstall(name='zerofree') + + +def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): + utils.LogStep('Create Image') + utils.Run(['rm', '-f', image_path]) + utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) + utils.Run(['parted', image_path, 'mklabel', 'msdos']) + utils.Run(['parted', image_path, 'mkpart', 'primary', + fs_type, '1', str(int(size_gb) * 1024)]) + + +def FormatImage(image_mapping_path): + utils.LogStep('Format Image') + utils.Run(['mkfs', image_mapping_path]) + utils.Sync() + + +def InstallArchLinux(base_dir): + utils.LogStep('Install Arch Linux') + utils.Pacstrap(base_dir, IMAGE_PACKAGES) + + +def SetupFileSystem(base_dir, image_mapping_path, fs_type): + utils.LogStep('File Systems') + _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], + capture_output=True) + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) + _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', + '-o', 'value', + image_mapping_path], + capture_output=True) + disk_uuid = disk_uuid.strip() + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), + 'UUID=%s / %s defaults 0 1' % (disk_uuid, fs_type)) + utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) + return disk_uuid + + +def PurgeDisk(mount_path): + paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] + for path in paths: + utils.DeleteDirectory(os.path.join(mount_path, path)) + + +def ShrinkDisk(image_mapping_path): + utils.LogStep('Shrink Disk') + utils.Run(['zerofree', image_mapping_path]) + + +def SaveImage(arch_root, image_filename): + utils.LogStep('Save Arch Linux Image in GCE format') + source_image_raw = os.path.join(arch_root, 'disk.raw') + image_raw = os.path.join(os.getcwd(), 'disk.raw') + image_file = os.path.join(os.getcwd(), image_filename) + utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) + utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) + return image_file + + +def UploadImage(image_path, gs_path, make_public=False): + utils.LogStep('Upload Image to Cloud Storage') + utils.SecureDeleteFile('~/.gsutil/*.url') + utils.Run(['gsutil', 'rm', gs_path]) + utils.Run(['gsutil', 'cp', image_path, gs_path]) + if make_public: + utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) + + +def AddImageToComputeEngineProject(image_name, gs_path, description): + utils.LogStep('Add image to project') + utils.Run( + ['gcloud', 'compute', 'images', 'delete', image_name, '-q'], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + utils.Run( + ['gcloud', 'compute', 'images', 'create', image_name, '-q', + '--source-uri', gs_path, + '--description', description], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + + +def GetImageNameAndDescription(outfile_name): + today = date.today() + isodate = today.strftime("%Y-%m-%d") + yyyymmdd = today.strftime("%Y%m%d") + image_name = 'arch-v%s' % yyyymmdd + if outfile_name: + image_filename = outfile_name + else: + image_filename = '%s.tar.gz' % image_name + description = 'Arch Linux x86-64 built on %s' % isodate + return image_name, image_filename, description + + def ParseArgs(): parser = argparse.ArgumentParser( description='Arch Linux Image Builder for Compute Engine') @@ -69,14 +221,6 @@ def ParseArgs(): dest='packages', nargs='+', help='Additional packages to install via Pacman.') - parser.add_argument('--mirror', - dest='mirror', - default=DEFAULT_MIRROR, - help='Mirror to download packages from.') - parser.add_argument('--bootstrap', - dest='bootstrap', - help='Arch Linux Bootstrap tarball. ' - '(default: Download latest version)') parser.add_argument('-v', '--verbose', dest='verbose', default=False, @@ -132,113 +276,12 @@ def ParseArgs(): default=False, help='Disables signature checking for pacman packages.', action='store_true') + parser.add_argument('--fs_type', + dest='fs_type', + default='ext4', + help='Verbose console output.', + action='store_true') return parser.parse_args() -def DownloadArchBootstrap(bootstrap_tarball): - utils.LogStep('Download Arch Linux Bootstrap') - if bootstrap_tarball: - url = bootstrap_tarball - sha1sum = None - else: - url, sha1sum = GetLatestBootstrapUrl() - logging.debug('Downloading %s', url) - local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url)) - if os.path.isfile(local_bootstrap): - logging.debug('Using local file instead.') - if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum: - return local_bootstrap - utils.DownloadFile(url, local_bootstrap) - if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum: - raise ValueError('Bad checksum') - return local_bootstrap - - -def ChrootIntoArchAndBuild(arch_root, relative_builder_path, args): - params = { - 'quiet': args.quiet, - 'verbose': args.verbose, - 'packages': args.packages, - 'mirror': args.mirror, - 'accounts': args.accounts, - 'debugmode': args.debug, - 'size_gb': args.size_gb - } - chroot_archenv_script = os.path.join('/', relative_builder_path, - 'arch-staging.py') - utils.RunChroot(arch_root, - '%s "%s"' % (chroot_archenv_script, utils.EncodeArgs(params))) - logging.debug('Bootstrap Chroot: sudo %s/bin/arch-chroot %s/', - arch_root, arch_root) - - -def SaveImage(arch_root, image_filename): - utils.LogStep('Save Arch Linux Image in GCE format') - source_image_raw = os.path.join(arch_root, 'disk.raw') - image_raw = os.path.join(os.getcwd(), 'disk.raw') - image_file = os.path.join(os.getcwd(), image_filename) - utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) - utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) - return image_file - - -def UploadImage(image_path, gs_path, make_public=False): - utils.LogStep('Upload Image to Cloud Storage') - utils.SecureDeleteFile('~/.gsutil/*.url') - utils.Run(['gsutil', 'rm', gs_path]) - utils.Run(['gsutil', 'cp', image_path, gs_path]) - if make_public: - utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) - - -def AddImageToComputeEngineProject(image_name, gs_path, description): - utils.LogStep('Add image to project') - utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) - utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q', - '--source-uri', gs_path, - '--description', description]) - -def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): - utils.LogStep('Setting up Bootstrap Environment') - arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0]) - mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server) - utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'), - mirrorlist) - utils.CreateDirectory(os.path.join(arch_root, 'run/shm')) - if use_pacman_keys: - utils.RunChroot(arch_root, 'pacman-key --init') - utils.RunChroot(arch_root, 'pacman-key --populate archlinux') - else: - utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') - # Install the most basic utilities for the bootstrapper. - utils.RunChroot(arch_root, - 'pacman --noconfirm -Sy python3 sed') - - return arch_root - - -def GetLatestBootstrapUrl(): - base_url = 'http://mirrors.kernel.org/archlinux/iso/latest/' - sha1sums = utils.HttpGet(base_url + 'sha1sums.txt') - items = sha1sums.splitlines() - for item in items: - if TARGET_ARCH in item and 'bootstrap' in item: - entries = item.split() - return base_url + entries[1], entries[0] - raise RuntimeError('Cannot find Arch bootstrap url') - - -def GetImageNameAndDescription(outfile_name): - today = date.today() - isodate = today.strftime("%Y-%m-%d") - yyyymmdd = today.strftime("%Y%m%d") - image_name = 'arch-v%s' % yyyymmdd - if outfile_name: - image_filename = outfile_name - else: - image_filename = '%s.tar.gz' % image_name - description = 'Arch Linux x86-64 built on %s' % isodate - return image_name, image_filename, description - - main() diff --git a/utils.py b/utils.py index a98c63c..f2d853f 100644 --- a/utils.py +++ b/utils.py @@ -80,14 +80,22 @@ def UserExists(username): return False -def CreateBuildUser(): - if not UserExists(BUILDER_USER): - Run(['useradd', BUILDER_USER, '-d', '/home/%s' % BUILDER_USER]) +def CreateBuildUser(user=BUILDER_USER): + if not UserExists(user): + home_dir = '/home/%s' % user + Run(['useradd', user, '-d', home_dir]) + Run(['mkdir', home_dir]) + Run(['chown', '%s:%s' % (user, user), home_dir]) + + +def RemoveBuildUser(): + if UserExists(BUILDER_USER): + Run(['userdel', '-r', BUILDER_USER]) def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): try: - logging.debug(params) + logging.debug('Run: %s in %s', params, cwd) if env: new_env = os.environ.copy() new_env.update(env) @@ -165,8 +173,13 @@ def DeleteFileFunc(file_pattern, delete_func): delete_func(item) +def DirectoryExists(dir_path): + return os.path.exists(dir_path) + + def DeleteDirectory(dir_path): - shutil.rmtree(dir_path) + if DirectoryExists(dir_path): + shutil.rmtree(dir_path) def CreateTempDirectory(base_dir=None): @@ -251,11 +264,11 @@ def ChangeDirectoryOwner(username, directory): def AurInstall(name=None, pkbuild_url=None): CreateBuildUser() if name: - pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower()) + pkbuild_url = 'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=%s' % (name.lower()) workspace_dir = CreateTempDirectory() DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) ChangeDirectoryOwner(BUILDER_USER, workspace_dir) - Run(['runuser', '-l', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) + Run(['runuser', '-m', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) tarball = tarball[0] Pacman(['-U', tarball], cwd=workspace_dir) @@ -267,7 +280,6 @@ def Pacstrap(base_dir, params): def Pacman(params, cwd=None): - #, '--debug' Run(['pacman', '--noconfirm'] + params, cwd=cwd) -- cgit v1.3 From 616a5802c3ba161ce292061af684c9618586ddd3 Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 3 Sep 2015 00:25:06 -0700 Subject: Removed the staging phase because the script will only work on Arch Linux now. This will make it much simpler to maintain. --- arch-staging.py | 282 -------------------------------------------------------- 1 file changed, 282 deletions(-) delete mode 100755 arch-staging.py diff --git a/arch-staging.py b/arch-staging.py deleted file mode 100755 index fbbb7a2..0000000 --- a/arch-staging.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import os -import logging -import sys -from datetime import date - -import utils - -COMPUTE_IMAGE_PACKAGES_GIT_URL = ( - 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') -IMAGE_FILE='disk.raw' -SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() -SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' - 'fakeroot arch-install-scripts').split() -IMAGE_PACKAGES = ('base tar wget ' - 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' - 'ntp psmisc openssh udev less bash-completion zip unzip ' - 'python2 python3').split() - - -def main(): - args = ParseArgs() - utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) - logging.info('Arch Linux Image Builder') - logging.info('========================') - - workspace_dir = None - image_file = None - try: - InstallPackagesForStagingEnvironment() - image_path = CreateArchImage(args) - image_name, image_filename, image_description = GetImageNameAndDescription( - args.outfile) - image_file = SaveImage(image_path, image_filename) - if args.upload and image_file: - UploadImage(image_file, args.upload, make_public=args.public) - if args.register: - AddImageToComputeEngineProject( - image_name, args.upload, image_description) - finally: - if not args.nocleanup and workspace_dir: - utils.DeleteDirectory(workspace_dir) - - -def CreateArchImage(args): - image_path = os.path.join(os.getcwd(), IMAGE_FILE) - CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) - mount_path = utils.CreateTempDirectory(base_dir='/') - image_mapping = utils.ImageMapper(image_path, mount_path) - try: - image_mapping.Map() - primary_mapping = image_mapping.GetFirstMapping() - image_mapping_path = primary_mapping['path'] - FormatImage(image_mapping_path) - try: - image_mapping.Mount() - utils.CreateDirectory('/run/shm') - utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) - InstallArchLinux(mount_path) - disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) - ConfigureArchInstall( - args, mount_path, primary_mapping['parent'], disk_uuid) - utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) - PurgeDisk(mount_path) - finally: - image_mapping.Unmount() - ShrinkDisk(image_mapping_path) - finally: - image_mapping.Unmap() - utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) - utils.Sync() - return image_path - - -def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): - relative_builder_path = utils.CopyBuilder(mount_path) - utils.LogStep('Download compute-image-packages') - packages_dir = utils.CreateTempDirectory(mount_path) - utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) - utils.CreateDirectory(os.path.join(mount_path, '')) - packages_dir = os.path.relpath(packages_dir, mount_path) - params = { - 'packages_dir': '/%s' % packages_dir, - 'device': parent_path, - 'disk_uuid': disk_uuid, - 'accounts': args.accounts, - 'debugmode': args.debug, - 'quiet': args.quiet, - 'verbose': args.verbose, - 'packages': args.packages, - 'size_gb': args.size_gb - } - config_arch_py = os.path.join( - '/', relative_builder_path, 'arch-image.py') - utils.RunChroot(mount_path, - '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), - use_custom_path=False) - utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) - - -def InstallPackagesForStagingEnvironment(): - utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) - utils.InstallPackages(SETUP_PACKAGES) - utils.RemoveBuildUser() - utils.AurInstall(name='multipath-tools-git') - utils.AurInstall(name='zerofree') - - -def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): - utils.LogStep('Create Image') - utils.Run(['rm', '-f', image_path]) - utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) - utils.Run(['parted', image_path, 'mklabel', 'msdos']) - utils.Run(['parted', image_path, 'mkpart', 'primary', - fs_type, '1', str(int(size_gb) * 1024)]) - - -def FormatImage(image_mapping_path): - utils.LogStep('Format Image') - utils.Run(['mkfs', image_mapping_path]) - utils.Sync() - - -def InstallArchLinux(base_dir): - utils.LogStep('Install Arch Linux') - utils.Pacstrap(base_dir, IMAGE_PACKAGES) - - -def SetupFileSystem(base_dir, image_mapping_path, fs_type): - utils.LogStep('File Systems') - _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], - capture_output=True) - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) - _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', - '-o', 'value', - image_mapping_path], - capture_output=True) - disk_uuid = disk_uuid.strip() - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), - 'UUID=%s / %s defaults 0 1' % (disk_uuid, fs_type)) - utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) - return disk_uuid - - -def PurgeDisk(mount_path): - paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] - for path in paths: - utils.DeleteDirectory(os.path.join(mount_path, path)) - - -def ShrinkDisk(image_mapping_path): - utils.LogStep('Shrink Disk') - utils.Run(['zerofree', image_mapping_path]) - - -def SaveImage(arch_root, image_filename): - utils.LogStep('Save Arch Linux Image in GCE format') - source_image_raw = os.path.join(arch_root, 'disk.raw') - image_raw = os.path.join(os.getcwd(), 'disk.raw') - image_file = os.path.join(os.getcwd(), image_filename) - utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) - utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) - return image_file - - -def UploadImage(image_path, gs_path, make_public=False): - utils.LogStep('Upload Image to Cloud Storage') - utils.SecureDeleteFile('~/.gsutil/*.url') - utils.Run(['gsutil', 'rm', gs_path]) - utils.Run(['gsutil', 'cp', image_path, gs_path]) - if make_public: - utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) - - -def AddImageToComputeEngineProject(image_name, gs_path, description): - utils.LogStep('Add image to project') - utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) - utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q', - '--source-uri', gs_path, - '--description', description]) - - -def GetImageNameAndDescription(outfile_name): - today = date.today() - isodate = today.strftime("%Y-%m-%d") - yyyymmdd = today.strftime("%Y%m%d") - image_name = 'arch-v%s' % yyyymmdd - if outfile_name: - image_filename = outfile_name - else: - image_filename = '%s.tar.gz' % image_name - description = 'Arch Linux x86-64 built on %s' % isodate - return image_name, image_filename, description - - -def ParseArgs(): - parser = argparse.ArgumentParser( - description='Arch Linux Image Builder for Compute Engine') - parser.add_argument('-p', '--packages', - dest='packages', - nargs='+', - help='Additional packages to install via Pacman.') - parser.add_argument('-v', '--verbose', - dest='verbose', - default=False, - help='Verbose console output.', - action='store_true') - parser.add_argument('-q', '--quiet', - dest='quiet', - default=False, - help='Suppress all console output.', - action='store_true') - parser.add_argument('--upload', - dest='upload', - default=None, - help='Google Cloud Storage path to upload to.') - parser.add_argument('--size_gb', - dest='size_gb', - default=10, - help='Volume size of image (in GiB).') - parser.add_argument('--accounts', - dest='accounts', - nargs='+', - help='Space delimited list of user accounts to create on ' - 'the image. Format: username:password') - parser.add_argument('--nocleanup', - dest='nocleanup', - default=False, - help='Prevent cleaning up the image build workspace ' - 'after image has been created.', - action='store_true') - parser.add_argument('--outfile', - dest='outfile', - default=None, - help='Name of the output image file.') - parser.add_argument('--debug', - dest='debug', - default=False, - help='Configure the image for debugging.', - action='store_true') - parser.add_argument('--public', - dest='public', - default=False, - help='Make image file uploaded to Cloud Storage ' - 'available for everyone.', - action='store_true') - parser.add_argument('--register', - dest='register', - default=False, - help='Add the image to Compute Engine project. ' - '(Upload to Cloud Storage required.)', - action='store_true') - parser.add_argument('--nopacmankeys', - dest='nopacmankeys', - default=False, - help='Disables signature checking for pacman packages.', - action='store_true') - parser.add_argument('--fs_type', - dest='fs_type', - default='ext4', - help='Verbose console output.', - action='store_true') - return parser.parse_args() - -main() -- cgit v1.3 From 43d132fa18791539f3c79ad9ebb1f4e3bc63109f Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 3 Sep 2015 00:32:15 -0700 Subject: Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0c92971..b7670a2 100644 --- a/README.md +++ b/README.md @@ -60,4 +60,4 @@ All files in this repository are under the [Apache License, Version 2.0](LICENSE ## Disclaimer -Google Inc. does not provide any support or guarnatees for this project or the images provided. +Google Inc. does not provide any support or guarantees for this project or the images provided. -- cgit v1.3 From d79b09e8ad4d64008aceaeb789e884098cfc2c9a Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 3 Sep 2015 07:36:48 -0700 Subject: Fix upload and publish. --- build-gce-arch.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/build-gce-arch.py b/build-gce-arch.py index 5a3e2b8..f64dfb3 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -183,22 +183,25 @@ def SaveImage(arch_root, image_filename): def UploadImage(image_path, gs_path, make_public=False): utils.LogStep('Upload Image to Cloud Storage') utils.SecureDeleteFile('~/.gsutil/*.url') - utils.Run(['gsutil', 'rm', gs_path]) - utils.Run(['gsutil', 'cp', image_path, gs_path]) + utils.Run(['gsutil', 'rm', gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + utils.Run(['gsutil', 'cp', image_path, gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) if make_public: - utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) + utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) def AddImageToComputeEngineProject(image_name, gs_path, description): utils.LogStep('Add image to project') utils.Run( - ['gcloud', 'compute', 'images', 'delete', image_name, '-q'], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + ['gcloud', 'compute', 'images', 'delete', image_name, '-q'], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) utils.Run( - ['gcloud', 'compute', 'images', 'create', image_name, '-q', - '--source-uri', gs_path, - '--description', description], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + ['gcloud', 'compute', 'images', 'create', image_name, '-q', + '--source-uri', gs_path, + '--description', description], + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) def GetImageNameAndDescription(outfile_name): -- cgit v1.3