From 52521dc682d675cb19bea34c5fdd64fdd3a37934 Mon Sep 17 00:00:00 2001 From: Jeremy Edwards Date: Thu, 3 Dec 2015 15:43:43 -0800 Subject: Update builder to reflect changes to Arch build process. --- README.md | 51 ++++++++++++++++++++++++++------------------------- arch-image.py | 11 +++++++++-- aurinstall.sh | 18 ++++++++++++++++++ build-gce-arch.py | 42 ++++++++++++++++++++++++------------------ push.sh | 17 +++++++++++++++++ utils.py | 20 +++++++++++++++++++- 6 files changed, 113 insertions(+), 46 deletions(-) create mode 100755 aurinstall.sh create mode 100755 push.sh diff --git a/README.md b/README.md index b7670a2..c9be8fb 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,29 @@ ## Arch Linux Image Builder for GCE -Creates an Arch Linux image that can run on Google Compute Engine. +This project is a collection of scripts that create an Arch Linux OS image that +can run on [Google Compute Engine](https://cloud.google.com/compute/). -The image is configured close to the recommendations listed on [Building an image from scratch](https://developers.google.com/compute/docs/images#buildingimage). +The image is configured close to the recommendations listed on +[Building an image from scratch](https://developers.google.com/compute/docs/images#buildingimage). These scripts are written in Python3. -## Premade Image -arch-v20150903 - [gs://jeremyje/archlinux-images/arch-v20150903.tar.gz](https://storage.googleapis.com/jeremyje/archlinux-images/arch-v20150903.tar.gz) +## Prebuilt Images + + * arch-v20151023 - [gs://jeremyje/archlinux-images/arch-v20151023.tar.gz](https://storage.googleapis.com/jeremyje/archlinux-images/arch-v20151023.tar.gz) + * arch-v20150903 - [gs://jeremyje/archlinux-images/arch-v20150903.tar.gz](https://storage.googleapis.com/jeremyje/archlinux-images/arch-v20150903.tar.gz) + +You can add these images using the +[Developers Console](https://console.developers.google.com/compute/imagesAdd). + +You can use [Cloud SDK](https://cloud.google.com/sdk/) to add the prebuilt +images to your project. To do that run the following command. + +``` +gcloud compute images insert arch-v20151023 \ + --source-uri gs://jeremyje/archlinux-images/arch-v20151023.tar.gz \ + --description "Arch Linux built on 2015-10-23" +``` ## Usage @@ -33,31 +49,16 @@ gsutil ls gs:// gsutil mb gs://${BUCKET} ``` -### Locally -``` -# Install Required Packages -# Arch Linux -sudo pacman -S python haveged - -./build-gce-arch.py --verbose -# Upload to Cloud Storage -gsutil cp archlinux-gce.tar.gz gs://${BUCKET}/archlinux.tar.gz - -# Add image to project -gcloud compute images insert archlinux \ - --source-uri gs://${BUCKET}/archlinux.tar.gz \ - --description "Arch Linux for Compute Engine" -``` - - -## Contributing changes +## Contributing Changes * See [CONTRIB.md](CONTRIB.md) ## Licensing -All files in this repository are under the [Apache License, Version 2.0](LICENSE) unless noted otherwise. +All files in this repository are under the +[Apache License, Version 2.0](LICENSE) unless noted otherwise. -## Disclaimer -Google Inc. does not provide any support or guarantees for this project or the images provided. +## Support +Google Inc. does not provide any support, guarantees, or warranty for this +project or the images provided. diff --git a/arch-image.py b/arch-image.py index e6cbb04..35ac5bd 100755 --- a/arch-image.py +++ b/arch-image.py @@ -242,7 +242,6 @@ class Arch(linux.LinuxPlatform): super(Arch, self).__init__() ''' - def main(): args = utils.DecodeArgs(sys.argv[1]) utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) @@ -257,6 +256,7 @@ def main(): SetupSsh() #SetupFail2ban() SetupAccounts(args) + #InstallImportedPackages(args['packages_dir']) InstallGcePackages(args['packages_dir']) ConfigMessageOfTheDay() ConfigureSecurity() @@ -432,6 +432,12 @@ def ConfigureSerialPortOutput(): utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1') +def InstallImportedPackages(packages_dir): + aur_packages_dir = os.path.join(packages_dir, 'aur') + for aur_package in os.listdir(aur_packages_dir): + utils.Pacman('-U', aur_package, cwd=aur_packages_dir) + + def InstallGcePackages(packages_dir): try: InstallGoogleCloudSdk() @@ -519,4 +525,5 @@ def ConfigMessageOfTheDay(): utils.WriteFile('/etc/motd', ETC_MOTD) -main() +if __name__ == '__main__': + main() diff --git a/aurinstall.sh b/aurinstall.sh new file mode 100755 index 0000000..d8582c6 --- /dev/null +++ b/aurinstall.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +InstallFromAur() { + local package_url="https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=$1" + local working_dir=${PWD} + local temp_dir=`mktemp -d` + cd ${temp_dir} + wget -O ${temp_dir}/PKGBUILD ${package_url} -nv + makepkg + sudo pacman -U `ls *.tar*` + rm ${temp_dir} -rf + cd ${working_dir} +} + +for package in "$@" +do + InstallFromAur "${package}" +done \ No newline at end of file diff --git a/build-gce-arch.py b/build-gce-arch.py index f64dfb3..cc80800 100755 --- a/build-gce-arch.py +++ b/build-gce-arch.py @@ -44,8 +44,8 @@ def main(): workspace_dir = None image_file = None try: - InstallPackagesForStagingEnvironment() - image_path = CreateArchImage(args) + aur_packages = InstallPackagesOnHostMachine() + image_path = CreateArchImage(args, aur_packages) image_name, image_filename, image_description = GetImageNameAndDescription( args.outfile) image_file = SaveImage(image_path, image_filename) @@ -59,12 +59,13 @@ def main(): utils.DeleteDirectory(workspace_dir) -def CreateArchImage(args): +def CreateArchImage(args, aur_packages): image_path = os.path.join(os.getcwd(), IMAGE_FILE) CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) mount_path = utils.CreateTempDirectory(base_dir='/') image_mapping = utils.ImageMapper(image_path, mount_path) try: + image_mapping.InstallLoopback() image_mapping.Map() primary_mapping = image_mapping.GetFirstMapping() image_mapping_path = primary_mapping['path'] @@ -76,7 +77,7 @@ def CreateArchImage(args): InstallArchLinux(mount_path) disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) ConfigureArchInstall( - args, mount_path, primary_mapping['parent'], disk_uuid) + args, mount_path, primary_mapping['parent'], disk_uuid, aur_packages) utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) PurgeDisk(mount_path) finally: @@ -89,12 +90,14 @@ def CreateArchImage(args): return image_path -def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): +def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid, aur_packages): relative_builder_path = utils.CopyBuilder(mount_path) - utils.LogStep('Download compute-image-packages') packages_dir = utils.CreateTempDirectory(mount_path) utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) utils.CreateDirectory(os.path.join(mount_path, '')) + aur_packages_dir = os.path.join(packages_dir, 'aur') + for aur_package in aur_packages: + utils.CopyFiles(aur_package, aur_packages_dir + '/') packages_dir = os.path.relpath(packages_dir, mount_path) params = { 'packages_dir': '/%s' % packages_dir, @@ -115,12 +118,16 @@ def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) -def InstallPackagesForStagingEnvironment(): +def InstallPackagesOnHostMachine(): + aur_packages = [] + utils.UpdatePacmanDatabase() utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) utils.InstallPackages(SETUP_PACKAGES) - utils.RemoveBuildUser() - utils.AurInstall(name='multipath-tools-git') - utils.AurInstall(name='zerofree') + utils.UpdateAllPackages() + aur_packages.append(utils.AurInstall(name='multipath-tools-git')) + aur_packages.append(utils.AurInstall(name='zerofree')) + aur_packages.append(utils.AurInstall(name='python2-crcmod')) + return aur_packages def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): @@ -170,14 +177,12 @@ def ShrinkDisk(image_mapping_path): utils.Run(['zerofree', image_mapping_path]) -def SaveImage(arch_root, image_filename): +def SaveImage(disk_image_file, image_filename): utils.LogStep('Save Arch Linux Image in GCE format') - source_image_raw = os.path.join(arch_root, 'disk.raw') - image_raw = os.path.join(os.getcwd(), 'disk.raw') - image_file = os.path.join(os.getcwd(), image_filename) - utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) - utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) - return image_file + image_raw = os.path.join(os.getcwd(), IMAGE_FILE) + gce_image_file = os.path.join(os.getcwd(), image_filename) + utils.Run(['tar', '-Szcf', image_filename, IMAGE_FILE]) + return gce_image_file def UploadImage(image_path, gs_path, make_public=False): @@ -287,4 +292,5 @@ def ParseArgs(): return parser.parse_args() -main() +if __name__ == '__main__': + main() diff --git a/push.sh b/push.sh new file mode 100755 index 0000000..7faa2dc --- /dev/null +++ b/push.sh @@ -0,0 +1,17 @@ +#!/bin/bash + + +VM_USER="${USER}" +PACKAGE_FILE="archbuilder.tar.gz" +INSTANCE_NAME="instance-1" +ZONE="us-east1-d" +ARCH_DATE="20151023" +SSH_TARGET=${VM_USER}@${INSTANCE_NAME} + +rm -f ${PACKAGE_FILE} +tar czf ${PACKAGE_FILE} * +gcloud compute ssh ${SSH_TARGET} --command "rm -fr *" --zone=${ZONE} +gcloud compute copy-files ${PACKAGE_FILE} ${SSH_TARGET}:/home/${VM_USER} --zone=${ZONE} + +gcloud compute ssh ${SSH_TARGET} --command "tar xvzf ${PACKAGE_FILE}; rm ${PACKAGE_FILE}; chmod +x *.sh" --zone=${ZONE} +gcloud compute ssh ${SSH_TARGET} --command "sudo ./build-gce-arch.py --verbose --size_gb=100 --debug --public --upload gs://jeremyje/archlinux-images/arch-v${ARCH_DATE}.tar.gz --register" --zone=${ZONE} diff --git a/utils.py b/utils.py index f2d853f..76156fd 100644 --- a/utils.py +++ b/utils.py @@ -30,7 +30,7 @@ import urllib.request, urllib.error, urllib.parse APP_NAME = 'archbuilder' -BUILDER_USER = APP_NAME +BUILDER_USER = 'nobody' ETC_LOCALE_GEN = ''' en_US.UTF-8 UTF-8 en_US ISO-8859-1 @@ -81,6 +81,9 @@ def UserExists(username): def CreateBuildUser(user=BUILDER_USER): + if user == BUILDER_USER: + logging.warning('Skipping create user.') + return if not UserExists(user): home_dir = '/home/%s' % user Run(['useradd', user, '-d', home_dir]) @@ -89,6 +92,9 @@ def CreateBuildUser(user=BUILDER_USER): def RemoveBuildUser(): + if True: + logging.warning('Skipping delete user.') + return if UserExists(BUILDER_USER): Run(['userdel', '-r', BUILDER_USER]) @@ -272,6 +278,8 @@ def AurInstall(name=None, pkbuild_url=None): tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) tarball = tarball[0] Pacman(['-U', tarball], cwd=workspace_dir) + + RemoveBuildUser() return tarball @@ -283,6 +291,13 @@ def Pacman(params, cwd=None): Run(['pacman', '--noconfirm'] + params, cwd=cwd) +def UpdatePacmanDatabase(): + Pacman(['-Sy']) + + +def UpdateAllPackages(): + Pacman(['-Syyu']) + def InstallPackages(package_list): Pacman(['-S'] + package_list) @@ -305,6 +320,9 @@ class ImageMapper(object): if not self._device_map: self.LoadPartitions() + def InstallLoopback(self): + SudoRun(['modprobe', 'loop']) + def LoadPartitions(self): return_code, out, err = SudoRun(['kpartx', '-l', self._raw_disk], capture_output=True) # Expected Format -- cgit v1.3