From f0f7d6405556862932c5d2fa1f95097200585bdc Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 12 Feb 2015 08:25:53 +0000 Subject: Update scripts to work with new Arch install scripts. --- .gitignore | 2 ++ LICENSE | 1 - arch-image.py | 16 +++++---- arch-staging.py | 3 +- build-arch-on-gce-remote.sh | 53 ----------------------------- build-arch-on-gce.sh | 6 ++-- build-gce-arch.py | 2 +- gcevm-script-build-arch.sh | 82 +++++++++++++++++++++++++++++++++++++++++++++ utils.py | 35 ++++++++++++++++--- 9 files changed, 130 insertions(+), 70 deletions(-) delete mode 100755 build-arch-on-gce-remote.sh create mode 100755 gcevm-script-build-arch.sh diff --git a/.gitignore b/.gitignore index 951b277..02f1795 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ *.pyc *.raw __pycache__/* +.c9/* + diff --git a/LICENSE b/LICENSE index 15f9dd5..a4c5efd 100644 --- a/LICENSE +++ b/LICENSE @@ -200,4 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - \ No newline at end of file diff --git a/arch-image.py b/arch-image.py index 9401412..0d07044 100755 --- a/arch-image.py +++ b/arch-image.py @@ -261,6 +261,7 @@ def main(): ConfigMessageOfTheDay() ConfigureSecurity() ConfigureSerialPortOutput() + DisableUnusedServices() OptimizePackages() @@ -284,7 +285,7 @@ def OptimizePackages(): def SetupLocale(): - utils.LogStep('Set Locale to US English (UTF-8)') + utils.LogStep('Set Locale to US English (UTF-8)') utils.SetupArchLocale() @@ -307,9 +308,6 @@ def ConfigureKernel(): def InstallBootloader(device, uuid, debugmode): utils.LogStep('Install Syslinux bootloader') - ''' - utils.Run(['syslinux-install_update', '-i', '-a', '-m']) - ''' utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) utils.CreateDirectory('/boot/syslinux') utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') @@ -345,14 +343,20 @@ def InstallBootloader(device, uuid, debugmode): 'systemd.log_target=console', 'systemd.journald.forward_to_syslog=yes', 'systemd.journald.forward_to_kmsg=yes', - 'systemd.journald.forward_to_console=yes',] + 'systemd.journald.forward_to_console=yes', + 'initcall_debug', + 'printk.time=y', + 'init=/usr/lib/systemd/systemd-bootchart',] boot_params = ' '.join(boot_params) boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'APPEND root=', boot_spec) - +def DisableUnusedServices(): + utils.DisableService('getty@tty1.service') + utils.DisableService('graphical.target') + def ForwardSystemdToConsole(): utils.LogStep('Installing syslinux bootloader') utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes') diff --git a/arch-staging.py b/arch-staging.py index 344e7b7..ee659fb 100755 --- a/arch-staging.py +++ b/arch-staging.py @@ -25,7 +25,8 @@ COMPUTE_IMAGE_PACKAGES_GIT_URL = ( 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') IMAGE_FILE='disk.raw' SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() -SETUP_PACKAGES = 'pacman wget gcc make parted git setconf libaio sudo'.split() +SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' + 'fakeroot').split() IMAGE_PACKAGES = ('base tar wget ' 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' 'ntp psmisc openssh udev less bash-completion zip unzip ' diff --git a/build-arch-on-gce-remote.sh b/build-arch-on-gce-remote.sh deleted file mode 100755 index 5cd584c..0000000 --- a/build-arch-on-gce-remote.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build an Arch Linux image from within a GCE Debian VM. - -BUILDER_ROOT=/mnt/archongce/source -INSTANCE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-name) -ZONE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-zone) -SCRIPT_PARAMS=$(/usr/share/google/get_metadata_value attributes/script-params) -SCRIPT_PARAMS="--verbose --register ${SCRIPT_PARAMS}" -GIT_SOURCE_URI=$(/usr/share/google/get_metadata_value attributes/git-source-uri) -REMOTE_IMAGE=$(echo "i = '${SCRIPT_PARAMS}'.split(); print i[i.index('--upload') + 1]" | python) - -echo "Builder Root: ${BUILDER_ROOT}" -echo "Instance Name: ${INSTANCE_NAME}" -echo "Instance Zone: ${ZONE_NAME}" -echo "Source Git Repository: ${GIT_SOURCE_URI}" -echo "Remote Image: ${REMOTE_IMAGE}" -echo "Running script as:" -echo " ./build-gce-arch.py ${SCRIPT_PARAMS}" - -echo "Setup Builder Environment" -mkdir -p ${BUILDER_ROOT} - -echo "Updating Cloud SDK" -yes | gcloud components update - -echo "Installing Dependencies" -sudo apt-get update -sudo apt-get install -y -qq python3 haveged git - -echo "Getting source code..." -git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} - -cd ${BUILDER_ROOT} -haveged -w 1024 -gsutil rm ${REMOTE_IMAGE} -./build-gce-arch.py ${SCRIPT_PARAMS} -cat /var/log/syslog | grep -o "startupscript.*" > builder.log -gsutil cp builder.log ${REMOTE_IMAGE}.log -gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} diff --git a/build-arch-on-gce.sh b/build-arch-on-gce.sh index d1122e2..632586d 100755 --- a/build-arch-on-gce.sh +++ b/build-arch-on-gce.sh @@ -19,7 +19,7 @@ INSTANCE_ID=${RANDOM} INSTANCE_NAME=archbuilder${INSTANCE_ID} ZONE_NAME=us-central1-f MACHINE_TYPE=n1-standard-2 -GIT_SOURCE_URI=https://github.com/GoogleCloudPlatform/compute-archlinux-image-builder.git +GIT_SOURCE_URI=https://github.com/jeremyje/compute-archlinux-image-builder.git SCRIPT_PARAMS="$*" function GcloudNotConfiguredHelp() { @@ -46,10 +46,10 @@ function PrintHelp() { function DeployVm() { echo "Creating Instance, ${INSTANCE_NAME}" gcloud compute instances create ${INSTANCE_NAME} \ - --image debian-7-backports \ + --image ubuntu-14-10 \ --machine-type ${MACHINE_TYPE} \ --zone ${ZONE_NAME} \ - --metadata-from-file startup-script=build-arch-on-gce-remote.sh \ + --metadata-from-file startup-script=gcevm-script-build-arch.sh \ --metadata \ script-params="${SCRIPT_PARAMS}" \ instance-name="${INSTANCE_NAME}" \ diff --git a/build-gce-arch.py b/build-gce-arch.py index ee487b6..f6d31c4 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -212,7 +212,7 @@ def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') # Install the most basic utilities for the bootstrapper. utils.RunChroot(arch_root, - 'pacman --noconfirm -Sy python3') + 'pacman --noconfirm -Sy python3 sed') return arch_root diff --git a/gcevm-script-build-arch.sh b/gcevm-script-build-arch.sh new file mode 100755 index 0000000..8ddaf67 --- /dev/null +++ b/gcevm-script-build-arch.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build an Arch Linux image from within a GCE Debian VM. + +BUILDER_ROOT=/mnt/archongce/source +INSTANCE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-name) +ZONE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-zone) +SCRIPT_PARAMS=$(/usr/share/google/get_metadata_value attributes/script-params) +SCRIPT_PARAMS="--verbose --register ${SCRIPT_PARAMS}" +GIT_SOURCE_URI=$(/usr/share/google/get_metadata_value attributes/git-source-uri) +REMOTE_IMAGE=$(echo "i = '${SCRIPT_PARAMS}'.split(); print i[i.index('--upload') + 1]" | python) + +echo "Builder Root: ${BUILDER_ROOT}" +echo "Instance Name: ${INSTANCE_NAME}" +echo "Instance Zone: ${ZONE_NAME}" +echo "Source Git Repository: ${GIT_SOURCE_URI}" +echo "Remote Image: ${REMOTE_IMAGE}" +echo "Running script as:" +echo " ./build-gce-arch.py ${SCRIPT_PARAMS}" + +echo "Setup Builder Environment" +mkdir -p ${BUILDER_ROOT} + +echo "Updating Cloud SDK" +yes | gcloud components update + +function InstallDependenciesForDebian { + echo "Installing Dependencies (Debian)" + apt-get update + apt-get install -y -qq python3 haveged git +} + +function InstallDependenciesForRedhat { + echo "Installing Dependencies (Redhat)" + rpm -Uvh http://download.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm + yum install -y python3 haveged git +} +if [ -f /etc/redhat-release ] +then + InstallDependenciesForRedhat +else + InstallDependenciesForDebian +fi + +echo "Getting source code..." +git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} + +cd ${BUILDER_ROOT} +haveged -w 1024 +gsutil rm ${REMOTE_IMAGE} +./build-gce-arch.py ${SCRIPT_PARAMS} + +function SaveLogForRedhat { + journalctl > builder.log +} + +function SaveLogForDebian { + cat /var/log/syslog | grep -o "startupscript.*" > builder.log +} + +if [ -f /etc/redhat-release ] +then + SaveLogForRedhat +else + SaveLogForDebian +fi + +gsutil cp builder.log ${REMOTE_IMAGE}.log +gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} diff --git a/utils.py b/utils.py index dec05da..a98c63c 100644 --- a/utils.py +++ b/utils.py @@ -20,6 +20,7 @@ import hashlib import json import os import logging +import pwd import shutil import subprocess import sys @@ -29,6 +30,7 @@ import urllib.request, urllib.error, urllib.parse APP_NAME = 'archbuilder' +BUILDER_USER = APP_NAME ETC_LOCALE_GEN = ''' en_US.UTF-8 UTF-8 en_US ISO-8859-1 @@ -70,6 +72,19 @@ def SudoRun(params, cwd=None, capture_output=False): return Run(params, capture_output=capture_output) +def UserExists(username): + try: + pwd.getpwnam(username) + return True + except: + return False + + +def CreateBuildUser(): + if not UserExists(BUILDER_USER): + Run(['useradd', BUILDER_USER, '-d', '/home/%s' % BUILDER_USER]) + + def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): try: logging.debug(params) @@ -103,7 +118,7 @@ def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True def DownloadFile(url, file_path): - Run(['wget', '-O', file_path, url]) + Run(['wget', '-O', file_path, url, '-nv']) def HttpGet(url): @@ -207,9 +222,9 @@ def DebugBash(): def DebugPrintFile(file_path): - logging.info('==================================================================================') + logging.info('==============================================================') logging.info('File: %s', file_path) - logging.info('==================================================================================') + logging.info('==============================================================') Run(['cat', file_path]) @@ -218,19 +233,29 @@ def Sync(): def EnableService(service_name): - Run(['systemctl', 'enable', service_name]) + Run(['systemctl', 'enable', service_name]) + + +def DisableService(service_name): + Run(['systemctl', 'disable', service_name]) def Symlink(source_file, dest_file): Run(['ln', '-s', source_file, dest_file]) +def ChangeDirectoryOwner(username, directory): + SudoRun(['chown', '-R', username, directory]) + + def AurInstall(name=None, pkbuild_url=None): + CreateBuildUser() if name: pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower()) workspace_dir = CreateTempDirectory() DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) - Run(['makepkg', '--asroot'], cwd=workspace_dir) + ChangeDirectoryOwner(BUILDER_USER, workspace_dir) + Run(['runuser', '-l', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) tarball = tarball[0] Pacman(['-U', tarball], cwd=workspace_dir) -- cgit v1.3