diff options
| author | Lorenzo Castelli <lcastelli@google.com> | 2018-10-23 14:26:17 -0700 |
|---|---|---|
| committer | Lorenzo Castelli <lcastelli@google.com> | 2018-10-24 11:12:35 -0700 |
| commit | 3e72d05138bfed064b4100c0186c635eddddb577 (patch) | |
| tree | 7bc525311a3ede73de89a4a64bfe4b67ec351c13 | |
| parent | 419bf9d7c345a2478c78a1f1b664da3fb693fb0d (diff) | |
| download | compute-archlinux-image-builder-3e72d05138bfed064b4100c0186c635eddddb577.tar.xz | |
Brings the project up to date with a new bash implementation.
See the updated README for more information about images generated by
the new script.
| -rw-r--r-- | CONTRIB.md | 76 | ||||
| -rw-r--r-- | PKGBUILD | 31 | ||||
| -rw-r--r-- | README.md | 137 | ||||
| -rwxr-xr-x | arch-image.py | 531 | ||||
| -rwxr-xr-x | aurinstall.sh | 18 | ||||
| -rwxr-xr-x | build-arch-gce | 185 | ||||
| -rwxr-xr-x | build-arch-on-gce.sh | 76 | ||||
| -rwxr-xr-x | build-gce-arch.py | 296 | ||||
| -rwxr-xr-x | gcevm-script-build-arch.sh | 82 | ||||
| -rwxr-xr-x | push.sh | 16 | ||||
| -rw-r--r-- | utils.py | 388 |
11 files changed, 332 insertions, 1504 deletions
@@ -1,64 +1,28 @@ -# How to become a contributor and submit your own code +# How to Contribute -## Contributor License Agreements +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. -We'd love to accept your sample apps and patches! Before we can take them, we -have to jump a couple of legal hurdles. +## Contributor License Agreement -Please fill out either the individual or corporate Contributor License Agreement -(CLA). +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to <https://cla.developers.google.com/> to see +your current agreements on file or to sign a new one. - * If you are an individual writing original source code and you're sure you - own the intellectual property, then you'll need to sign an [individual CLA] - (https://developers.google.com/open-source/cla/individual). - * If you work for a company that wants to allow you to contribute your work, - then you'll need to sign a [corporate CLA] - (https://developers.google.com/open-source/cla/corporate). +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. -Follow either of the two links above to access the appropriate CLA and -instructions for how to sign and return it. Once we receive it, we'll be able to -accept your pull requests. +## Code reviews -## Contributing A Patch +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owner will respond to your issue promptly. -1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the desired repo, develop and test your code changes. -1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the - recommended coding standards for this organization. -1. Ensure that your code has an appropriate set of unit tests which all pass. -1. Submit a pull request. +## Community Guidelines -## Contributing A New Sample App - -1. Submit an issue to the GoogleCloudPlatform/Template repo describing your - proposed sample app. -1. The Template repo owner will respond to your enhancement issue promptly. - Instructional value is the top priority when evaluating new app proposals for - this collection of repos. -1. If your proposal is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Create your own repo for your app following this naming convention: - * {product}-{app-name}-{language} - * products: appengine, compute, storage, bigquery, prediction, cloudsql - * example: appengine-guestbook-python - * For multi-product apps, concatenate the primary products, like this: - compute-appengine-demo-suite-python. - * For multi-language apps, concatenate the primary languages like this: - appengine-sockets-python-java-go. - -1. Clone the README.md, CONTRIB.md and LICENSE files from the - GoogleCloudPlatform/Template repo. -1. Ensure that your code adheres to the existing style in the sample to which - you are contributing. Refer to the - [Google Cloud Platform Samples Style Guide] - (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the - recommended coding standards for this organization. -1. Ensure that your code has an appropriate set of unit tests which all pass. -1. Submit a request to fork your repo in GoogleCloudPlatform organizationt via - your proposal issue.
\ No newline at end of file +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..d1d808d --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,31 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Maintainer: Lorenzo Castelli <lcastelli@google.com> +# Maintainer: Samuel Littley <samuellittley@google.com> + +pkgname='build-arch-gce' +pkgver=0.1 +pkgrel=1 +pkgdesc='Builds a Arch image for Google Compute Engine' +arch=('any') +url='' +license=('Apache') +depends=('arch-install-scripts' 'e2fsprogs') +source=('build-arch-gce') +sha256sums=('7630868a98e3713bdf12fcfecd8733796b113296336c99e5827cb7b97e191a37') + +package() { + install -m755 -Dt "$pkgdir/usr/bin/" build-arch-gce +} @@ -1,67 +1,122 @@ -## Arch Linux Image Builder for GCE +## Arch Linux Image Builder for Google Compute Engine -This project is a collection of scripts that create an Arch Linux OS image that -can run on [Google Compute Engine](https://cloud.google.com/compute/). +This project provides a script that creates an [Arch +Linux](https://www.archlinux.org/) image that can run on [Google Compute +Engine](https://cloud.google.com/compute/). -The image is configured close to the recommendations listed on -[Building an image from scratch](https://developers.google.com/compute/docs/images#buildingimage). +The image is configured to be as close as possible to a base Arch Linux +installation, while still allowing it to be fully functional and optimized for +Compute Engine. Notable choices made and differences compared to a standard +Arch Linux installation are the following: -These scripts are written in Python3. +- GRUB is used with BIOS-based boot and a GPT partition table. +- Serial console logging is enabled from kernel command line and journald is + configured to forward to it. +- Block multiqueue and elevator noop are configured from kernel command line to + optimize Compute Engine disk performance. +- A minimal initcpio is configured for booting on Compute Engine virtual + machines. +- Root filesystem is ext4. +- Locale is set to en_US.UTF-8 and timezone is set to UTC. +- Network is configured through dhclient. +- Systemd-timesyncd is enabled and configured to use the Compute Engine metadata + server. +- Rng-tools are installed and enabled to provide entropy. +- Pacman keyring is configured to be built and initialized on first boot. +- Pacman mirror list is taken fresh from Arch Linux servers at the time the + image is built. +- [Linux Guest Environment for Google Compute + Engine](https://github.com/GoogleCloudPlatform/compute-image-packages) is + installed and enabled. +- An OpenSSH server is installed and enabled, with root login and password + authentication forbidden. User SSH keys are deployed and managed + automatically by the Linux Guest Environment as described in the + [corresponding + documentation](https://cloud.google.com/compute/docs/instances/connecting-to-instance). +- Sudo is installed. Permission to use sudo is managed automatically by Linux + Guest Environment. +- Root partition and filesystem are automatically extended at boot using + [growpart](https://launchpad.net/cloud-utils), to support dynamic disk + resizing. +- An additional Pacman repository is used to install and keep the Linux Guest + Environment and growpart packages up to date. -## Prebuilt Images - * arch-v20160502 - [gs://gce-arch-images/arch-v20160502.tar.gz](https://storage.googleapis.com/gce-arch-images/arch-v20160502.tar.gz) - * arch-v20151203 - [gs://gce-arch-images/arch-v20151203.tar.gz](https://storage.googleapis.com/gce-arch-images/arch-v20151203.tar.gz) - * arch-v20151103 - [gs://gce-arch-images/arch-v20151103.tar.gz](https://storage.googleapis.com/gce-arch-images/arch-v20151103.tar.gz) - * arch-v20151023 - [gs://gce-arch-images/arch-v20151023.tar.gz](https://storage.googleapis.com/gce-arch-images/arch-v20151023.tar.gz) - * arch-v20150903 - [gs://gce-arch-images/arch-v20150903.tar.gz](https://storage.googleapis.com/gce-arch-images/arch-v20150903.tar.gz) -You can add these images using the -[Developers Console](https://console.developers.google.com/compute/imagesAdd). +## Prebuilt Images -You can use [Cloud SDK](https://cloud.google.com/sdk/) to add the prebuilt -images to your project. To do that run the following command. +You can use [Cloud SDK](https://cloud.google.com/sdk/docs/) to create instances +with the latest prebuilt Arch Linux image. To do that follow the SDK +installation procedure, and then run the [following +command](https://cloud.google.com/sdk/gcloud/reference/compute/instances/create): -``` -gcloud compute images create arch-v20160502 \ - --source-uri gs://gce-arch-images/arch-v20160502.tar.gz \ - --description "Arch Linux built on 2016-05-02" - --family "arch" +```console +$ gcloud compute instances create INSTANCE_NAME \ + --image-project=arch-linux-gce --image-family=arch ``` -## Usage -### Install and Configure Cloud SDK (one time setup) -``` -# Install Cloud SDK (https://developers.google.com/cloud/sdk/) -# For linux: -curl https://sdk.cloud.google.com | bash +## Build Your Own Image -gcloud auth login -gcloud config set project <project> -# Your project ID in Cloud Console, https://console.developers.google.com/ -``` +You can build the Arch Linux image yourself with the following procedure: -### On a Compute Engine VM (recommended) -``` -./build-arch-on-gce.sh --upload gs://${BUCKET}/archlinux.tar.gz +1. Make sure you have required dependencies installed: + + ```console + $ sudo pacman -S arch-install-scripts e2fsprogs + ``` + +2. Run the image building script: + + ```console + $ sudo ./build-arch-gce + ``` + + If the script is successful, this will create an image file named + arch-vDATE.tar.gz in the current directory, where DATE is the current date. + +3. Install and configure the [Cloud SDK](https://cloud.google.com/sdk/docs/). -# You will need a Cloud Storage bucket. -# List buckets owned by your project. -gsutil ls gs:// -# Create a new bucket -gsutil mb gs://${BUCKET} +4. Copy the image file to Google Cloud Storage: + + ```console + $ gsutil mb gs://BUCKET_NAME + $ gsutil cp arch-vDATE.tar.gz gs://BUCKET_NAME + ``` + +5. Import the image file to Google Cloud Engine as a new custom image: + + ```console + $ gcloud compute images create IMAGE_NAME \ + --source-uri=gs://BUCKET_NAME/arch-vDATE.tar.gz \ + --guest-os-features=VIRTIO_SCSI_MULTIQUEUE + ``` + +You can now create new instances with your custom image: + +```console +$ gcloud compute instances create INSTANCE_NAME --image=IMAGE_NAME ``` +The Google Cloud Storage file is no longer needed, so you can delete if you +want: + + ```console + $ gsutil rm gs://BUCKET_NAME/arch-vDATE.tar.gz + ``` + + ## Contributing Changes * See [CONTRIB.md](CONTRIB.md) ## Licensing -All files in this repository are under the -[Apache License, Version 2.0](LICENSE) unless noted otherwise. + +All files in this repository are under the [Apache License, Version +2.0](LICENSE) unless noted otherwise. ## Support + Google Inc. does not provide any support, guarantees, or warranty for this project or the images provided. diff --git a/arch-image.py b/arch-image.py deleted file mode 100755 index 9fd9e8f..0000000 --- a/arch-image.py +++ /dev/null @@ -1,531 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import sys - -import utils - - -ETC_MOTD = '''Arch Linux for Compute Engine -''' - -ETC_HOSTS = '''127.0.0.1 localhost -169.254.169.254 metadata.google.internal metadata -''' - -ETC_SSH_SSH_CONFIG = ''' -Host * -Protocol 2 -ForwardAgent no -ForwardX11 no -HostbasedAuthentication no -StrictHostKeyChecking no -Ciphers aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc -Tunnel no - -# Google Compute Engine times out connections after 10 minutes of inactivity. -# Keep alive ssh connections by sending a packet every 7 minutes. -ServerAliveInterval 420 -''' - -ETC_SSH_SSHD_CONFIG = ''' -# Disable PasswordAuthentication as ssh keys are more secure. -PasswordAuthentication no - -# Disable root login, using sudo provides better auditing. -PermitRootLogin no - -PermitTunnel no -AllowTcpForwarding yes -X11Forwarding no - -# Compute times out connections after 10 minutes of inactivity. Keep alive -# ssh connections by sending a packet every 7 minutes. -ClientAliveInterval 420 - -# Restrict sshd to just IPv4 for now as sshd gets confused for things -# like X11 forwarding. - -Port 22 -Protocol 2 - -UsePrivilegeSeparation yes - -# Lifetime and size of ephemeral version 1 server key -KeyRegenerationInterval 3600 -ServerKeyBits 768 - -SyslogFacility AUTH -LogLevel INFO - -LoginGraceTime 120 -StrictModes yes - -RSAAuthentication yes -PubkeyAuthentication yes - -IgnoreRhosts yes -RhostsRSAAuthentication no -HostbasedAuthentication no - -PermitEmptyPasswords no -ChallengeResponseAuthentication no - -PasswordAuthentication no -PrintMotd no -PrintLastLog yes - -TCPKeepAlive yes - -Subsystem sftp /usr/lib/openssh/sftp-server - -UsePAM yes -UseDNS no -''' - -ETC_SYSCTL_D_70_DISABLE_IPV6_CONF = ''' -net.ipv6.conf.all.disable_ipv6 = 1 -''' - -ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF = ''' -# enables syn flood protection -net.ipv4.tcp_syncookies = 1 - -# ignores source-routed packets -net.ipv4.conf.all.accept_source_route = 0 - -# ignores source-routed packets -net.ipv4.conf.default.accept_source_route = 0 - -# ignores ICMP redirects -net.ipv4.conf.all.accept_redirects = 0 - -# ignores ICMP redirects -net.ipv4.conf.default.accept_redirects = 0 - -# ignores ICMP redirects from non-GW hosts -net.ipv4.conf.all.secure_redirects = 1 - -# ignores ICMP redirects from non-GW hosts -net.ipv4.conf.default.secure_redirects = 1 - -# don't allow traffic between networks or act as a router -net.ipv4.ip_forward = 0 - -# don't allow traffic between networks or act as a router -net.ipv4.conf.all.send_redirects = 0 - -# don't allow traffic between networks or act as a router -net.ipv4.conf.default.send_redirects = 0 - -# reverse path filtering - IP spoofing protection -net.ipv4.conf.all.rp_filter = 1 - -# reverse path filtering - IP spoofing protection -net.ipv4.conf.default.rp_filter = 1 - -# reverse path filtering - IP spoofing protection -net.ipv4.conf.default.rp_filter = 1 - -# ignores ICMP broadcasts to avoid participating in Smurf attacks -net.ipv4.icmp_echo_ignore_broadcasts = 1 - -# ignores bad ICMP errors -net.ipv4.icmp_ignore_bogus_error_responses = 1 - -# logs spoofed, source-routed, and redirect packets -net.ipv4.conf.all.log_martians = 1 - -# log spoofed, source-routed, and redirect packets -net.ipv4.conf.default.log_martians = 1 - -# implements RFC 1337 fix -net.ipv4.tcp_rfc1337 = 1 - -# randomizes addresses of mmap base, heap, stack and VDSO page -kernel.randomize_va_space = 2 -''' - -ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF = ''' -# provides protection from ToCToU races -fs.protected_hardlinks=1 - -# provides protection from ToCToU races -fs.protected_symlinks=1 - -# makes locating kernel addresses more difficult -kernel.kptr_restrict=1 - -# set ptrace protections -kernel.yama.ptrace_scope=1 - -# set perf only available to root -kernel.perf_event_paranoid=2 -''' - -ETC_PAM_D_PASSWD = ''' -#%PAM-1.0 -password required pam_cracklib.so difok=2 minlen=8 dcredit=2 ocredit=2 retry=3 -password required pam_unix.so sha512 shadow nullok -password required pam_tally.so even_deny_root_account deny=3 lock_time=5 unlock_time=3600 -''' - -ETC_SUDOERS_D_ADD_GROUP_ADM = ''' -%adm ALL=(ALL) ALL -''' - -ETC_FAIL2BAN_JAIL_LOCAL = ''' -[DEFAULT] -backend = systemd -loglevel = WARNING -''' - -ETC_FAIL2BAN_JAIL_D_SSHD_CONF = ''' -# fail2ban SSH -# block ssh after 3 unsuccessful login attempts for 10 minutes -[sshd] -enabled = true -action = iptables[chain=INPUT, protocol=tcp, port=22, name=sshd] -maxRetry = 3 -findtime = 600 -bantime = 600 -port = 22 -''' - -GCIMAGEBUNDLE_ARCH_PY = ''' -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Arch Linux specific platform info.""" - - -import os - -from gcimagebundlelib import linux - - -class Arch(linux.LinuxPlatform): - """Arch Linux specific information.""" - - @staticmethod - def IsThisPlatform(root='/'): - return os.path.isfile('/etc/arch-release') - - def __init__(self): - super(Arch, self).__init__() -''' - -def main(): - args = utils.DecodeArgs(sys.argv[1]) - utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) - logging.info('Setup Bootstrapper Environment') - SetupLocale() - ConfigureTimeZone() - ConfigureKernel() - InstallBootloader(args['device'], args['disk_uuid'], args['debugmode']) - ForwardSystemdToConsole() - SetupNtpServer() - SetupNetwork() - SetupSsh() - #SetupFail2ban() - SetupAccounts(args) - #InstallImportedPackages(args['packages_dir']) - InstallGcePackages(args['packages_dir']) - ConfigMessageOfTheDay() - ConfigureSecurity() - ConfigureSerialPortOutput() - DisableUnusedServices() - OptimizePackages() - - -def SetupAccounts(args): - accounts = args['accounts'] - if accounts: - utils.LogStep('Add Accounts') - for account in accounts: - username, password = account.split(':') - logging.info(' - %s', username) - utils.Run(['useradd', username, '-m', '-s', '/bin/bash', - '-G', 'adm,video']) - utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True) - - -def OptimizePackages(): - utils.LogStep('Cleanup Cached Package Data') - utils.Pacman(['-Syu']) - utils.Pacman(['-Sc']) - utils.Run(['pacman-optimize']) - - -def SetupLocale(): - utils.LogStep('Set Locale to US English (UTF-8)') - utils.SetupArchLocale() - - -def ConfigureTimeZone(): - utils.LogStep('Set Timezone to UTC') - utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime']) - - -def ConfigureKernel(): - utils.LogStep('Configure Kernel') - utils.Replace('/etc/mkinitcpio.conf', - 'MODULES=""', - 'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"') - utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '') - utils.Run(['mkinitcpio', - '-g', '/boot/initramfs-linux.img', - '-k', '/boot/vmlinuz-linux', - '-c', '/etc/mkinitcpio.conf']) - - -def InstallBootloader(device, uuid, debugmode): - utils.LogStep('Install Syslinux bootloader') - utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) - utils.CreateDirectory('/boot/syslinux') - utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') - utils.Run(['extlinux', '--install', '/boot/syslinux']) - utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1') - utils.Run(['fdisk', '-l', device]) - utils.Run(['dd', 'bs=440', 'count=1', 'conv=notrunc', - 'if=/usr/lib/syslinux/bios/mbr.bin', 'of=%s' % device]) - - boot_params = [ - 'console=ttyS0,38400', - 'CONFIG_KVM_GUEST=y', - 'CONFIG_KVM_CLOCK=y', - 'CONFIG_VIRTIO_PCI=y', - 'CONFIG_SCSI_VIRTIO=y', - 'CONFIG_VIRTIO_NET=y', - 'CONFIG_STRICT_DEVMEM=y', - 'CONFIG_DEVKMEM=n', - 'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536', - 'CONFIG_DEBUG_RODATA=y', - 'CONFIG_DEBUG_SET_MODULE_RONX=y', - 'CONFIG_CC_STACKPROTECTOR=y', - 'CONFIG_COMPAT_VDSO=n', - 'CONFIG_COMPAT_BRK=n', - 'CONFIG_X86_PAE=y', - 'CONFIG_SYN_COOKIES=y', - 'CONFIG_SECURITY_YAMA=y', - 'CONFIG_SECURITY_YAMA_STACKED=y', - ] - if debugmode: - boot_params += [ - 'systemd.log_level=debug', - 'systemd.log_target=console', - 'systemd.journald.forward_to_syslog=yes', - 'systemd.journald.forward_to_kmsg=yes', - 'systemd.journald.forward_to_console=yes',] - boot_params = ' '.join(boot_params) - boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) - utils.ReplaceLine('/boot/syslinux/syslinux.cfg', - 'APPEND root=', - boot_spec) - -def DisableUnusedServices(): - utils.DisableService('getty@tty1.service') - utils.DisableService('graphical.target') - -def ForwardSystemdToConsole(): - utils.LogStep('Installing syslinux bootloader') - utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes') - - -def SetupNtpServer(): - utils.LogStep('Configure NTP') - utils.WriteFile('/etc/ntp.conf', 'server metadata.google.internal iburst') - - -def SetupNetwork(): - utils.LogStep('Setup Networking') - utils.SecureDeleteFile('/etc/hostname') - utils.WriteFile('/etc/hosts', ETC_HOSTS) - utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf', - ETC_SYSCTL_D_70_DISABLE_IPV6_CONF) - # https://wiki.archlinux.org/index.php/Network_configuration#Reverting_to_traditional_device_names - utils.Symlink('/dev/null', '/etc/udev/rules.d/80-net-setup-link.rules') - utils.EnableService('dhcpcd.service') - utils.EnableService('systemd-networkd.service') - utils.EnableService('systemd-networkd-wait-online.service') - - -def SetupSsh(): - utils.LogStep('Configure SSH') - utils.WriteFile('/etc/ssh/sshd_not_to_be_run', 'GOOGLE') - utils.SecureDeleteFile('/etc/ssh/ssh_host_key') - utils.SecureDeleteFile('/etc/ssh/ssh_host_rsa_key*') - utils.SecureDeleteFile('/etc/ssh/ssh_host_dsa_key*') - utils.SecureDeleteFile('/etc/ssh/ssh_host_ecdsa_key*') - utils.WriteFile('/etc/ssh/ssh_config', ETC_SSH_SSH_CONFIG) - utils.Chmod('/etc/ssh/ssh_config', 644) - utils.WriteFile('/etc/ssh/sshd_config', ETC_SSH_SSHD_CONFIG) - utils.Chmod('/etc/ssh/sshd_config', 644) - utils.EnableService('sshd.service') - - -def SetupFail2ban(): - utils.LogStep('Configure fail2ban') - # http://flexion.org/posts/2012-11-ssh-brute-force-defence.html - utils.Pacman(['-S', 'fail2ban']) - utils.WriteFile('/etc/fail2ban/jail.local', ETC_FAIL2BAN_JAIL_LOCAL) - utils.WriteFile('/etc/fail2ban/jail.d/sshd.conf', - ETC_FAIL2BAN_JAIL_D_SSHD_CONF) - utils.EnableService('syslog-ng') - utils.EnableService('fail2ban.service') - - -def ConfigureSecurity(): - utils.LogStep('Compute Engine Security Recommendations') - utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf', - ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF) - utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf', - ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF) - utils.LogStep('Lock Root User Account') - utils.Run(['usermod', '-L', 'root']) - utils.LogStep('PAM Security Settings') - utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD) - - utils.LogStep('Disable CAP_SYS_MODULE') - utils.WriteFile('/proc/sys/kernel/modules_disabled', '1') - - utils.LogStep('Remove the kernel symbol table') - utils.SecureDeleteFile('/boot/System.map') - - utils.LogStep('Sudo Access') - utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM) - utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm']) - utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm']) - - -def ConfigureSerialPortOutput(): - # https://wiki.archlinux.org/index.php/working_with_the_serial_console - # Try this: http://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot - utils.LogStep('Configure Serial Port Output') - - utils.Sed('/boot/syslinux/syslinux.cfg', '/DEFAULT/aserial 0 38400') - utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1') - - -def InstallImportedPackages(packages_dir): - aur_packages_dir = os.path.join(packages_dir, 'aur') - for aur_package in os.listdir(aur_packages_dir): - utils.Pacman('-U', aur_package, cwd=aur_packages_dir) - - -def InstallGcePackages(packages_dir): - try: - InstallGoogleCloudSdk() - except: - pass - try: - InstallComputeImagePackages(packages_dir) - except: - pass - - -def InstallComputeImagePackages(packages_dir): - utils.LogStep('Install compute-image-packages') - utils.Run(["egrep -lRZ 'python' %s | " - "xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" % - packages_dir], - shell=True) - utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/') - utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'), - '/') - utils.SecureDeleteFile('/README.md') - # TODO: Fix gcimagebundle does not work with Arch yet. - #InstallGcimagebundle(packages_dir) - - # Patch Google services to run after the network is actually available. - PatchGoogleSystemdService( - '/usr/lib/systemd/system/google-startup-scripts.service') - PatchGoogleSystemdService( - '/usr/lib/systemd/system/google-accounts-manager.service') - PatchGoogleSystemdService( - '/usr/lib/systemd/system/google-address-manager.service') - PatchGoogleSystemdService( - '/usr/lib/systemd/system/google.service') - utils.EnableService('google-accounts-manager.service') - utils.EnableService('google-address-manager.service') - utils.EnableService('google.service') - utils.EnableService('google-startup-scripts.service') - utils.DeleteDirectory(packages_dir) - - -def InstallGcimagebundle(packages_dir): - utils.WriteFile( - os.path.join(packages_dir, 'gcimagebundle/gcimagebundlelib/arch.py'), - GCIMAGEBUNDLE_ARCH_PY) - utils.Run(['python2', 'setup.py', 'install'], - cwd=os.path.join(packages_dir, 'gcimagebundle')) - - -def PatchGoogleSystemdService(file_path): - utils.ReplaceLine(file_path, - 'After=network.target', 'After=network-online.target') - utils.ReplaceLine(file_path, - 'Requires=network.target', 'Requires=network-online.target') - - -def InstallGoogleCloudSdk(): - # TODO: There's a google-cloud-sdk in AUR which should be used - # but it's not optimal for cloud use. The image is too large. - utils.LogStep('Install Google Cloud SDK') - usr_share_google = '/usr/share/google' - archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip') - unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk') - utils.CreateDirectory(usr_share_google) - utils.DownloadFile( - 'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip', archive) - utils.Run(['unzip', archive, '-d', usr_share_google]) - utils.AppendFile('/etc/bash.bashrc', - 'export CLOUDSDK_PYTHON=/usr/bin/python2') - utils.Run([os.path.join(unzip_dir, 'install.sh'), - '--usage-reporting', 'false', - '--bash-completion', 'true', - '--disable-installation-options', - '--rc-path', '/etc/bash.bashrc', - '--path-update', 'true'], - cwd=unzip_dir, - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud') - utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil') - utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil') - utils.SecureDeleteFile(archive) - - -def ConfigMessageOfTheDay(): - utils.LogStep('Configure Message of the Day') - utils.WriteFile('/etc/motd', ETC_MOTD) - - -if __name__ == '__main__': - main() diff --git a/aurinstall.sh b/aurinstall.sh deleted file mode 100755 index 22656ae..0000000 --- a/aurinstall.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -InstallFromAur() { - local package_url="https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=$1" - local working_dir=${PWD} - local temp_dir=`mktemp -d` - cd ${temp_dir} - wget -O ${temp_dir}/PKGBUILD ${package_url} -nv - makepkg - sudo pacman -U `ls *.tar*` - rm ${temp_dir} -rf - cd ${working_dir} -} - -for package in "$@" -do - InstallFromAur "${package}" -done diff --git a/build-arch-gce b/build-arch-gce new file mode 100755 index 0000000..6768fa9 --- /dev/null +++ b/build-arch-gce @@ -0,0 +1,185 @@ +#!/bin/bash +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eEuo pipefail +trap 'echo "Error: \`$BASH_COMMAND\` exited with status $?"' ERR + +if (( EUID != 0 )); then + echo 'This script must be run with root privileges.' + exit 1 +fi + +# Setup cleanup trap to remove all temporary data. +cleanup() { + echo "- Cleaning up." + [[ ${mount_dir:-} ]] && umount "$mount_dir" + [[ ${loop_dev:-} ]] && losetup --detach "$loop_dev" + [[ ${work_dir:-} ]] && rm -r "$work_dir" + return 0 +} +trap cleanup EXIT + +echo "- Creating an empty raw disk image." +work_dir=$(mktemp --directory --tmpdir="$PWD" build-arch-gce.XXX) +disk_raw=$work_dir/disk.raw +truncate --size=10G -- "$disk_raw" + +echo "- Setting up a loop device and partitioning the image." +loop_dev=$(losetup --find --partscan --show -- "$disk_raw") +sfdisk --quiet -- "$loop_dev" <<-'EOF' + label:gpt + type=21686148-6449-6E6F-744E-656564454649,size=1MiB,attrs=LegacyBIOSBootable,name=bios_boot + type=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709,name=root +EOF + +echo "- Formatting the root partition." +root_dev=${loop_dev}p2 +mkfs.ext4 -q -L root -- "$root_dev" + +echo "- Mounting the root partition." +mount_dir=$work_dir/disk.mnt +mkdir -- "$mount_dir" +mount -- "$root_dev" "$mount_dir" + +echo "- Installing Arch Linux." +append_gce_repo() { + gawk -i inplace ' + /^\[gce\]$/ { found = 1 } { print } + ENDFILE { if (!found) { + print "" + print "[gce]" + print "Server = https://storage.googleapis.com/arch-linux-gce/repo" + print "SigLevel = Optional TrustAll" + } }' "$1" +} +cp /etc/pacman.conf "$work_dir" +append_gce_repo "$work_dir/pacman.conf" +pacstrap -G -M -C "$work_dir/pacman.conf" -- "$mount_dir" \ + base grub dhclient openssh sudo rng-tools google-compute-engine growpart +append_gce_repo "$mount_dir/etc/pacman.conf" + +echo "- Configuring fstab." +root_uuid=$(lsblk --noheadings --raw --output UUID -- "$root_dev") +{ + printf '# LABEL=%s\n' root + printf 'UUID=%-20s' "$root_uuid" + printf '\t%-10s' / ext4 defaults + printf '\t%s %s' 0 1 + printf '\n\n' +} >> "$mount_dir/etc/fstab" + +echo "- Running additional setup in chroot." +arch-chroot -- "$mount_dir" /bin/bash -s -- "$loop_dev" <<-'EOS' + set -eEuo pipefail + trap 'echo "Error: \`$BASH_COMMAND\` exited with status $?"' ERR + + echo "-- Configuring time." + ln -sf /usr/share/zoneinfo/UTC /etc/localtime + gawk -i assert -i inplace ' + /^#NTP=/ { $0 = "NTP=metadata.google.internal"; ++f } + { print } END { assert(f == 1, "f == 1") }' /etc/systemd/timesyncd.conf + systemctl --quiet enable systemd-timesyncd + + echo "-- Configuring locale." + gawk -i assert -i inplace ' + /^#en_US\.UTF-8 UTF-8\s*$/ { $0 = substr($0, 2); ++f } + { print } END { assert(f == 1, "f == 1") }' /etc/locale.gen + locale-gen + echo 'LANG=en_US.UTF-8' > /etc/locale.conf + + echo "-- Configuring journald." + gawk -i assert -i inplace ' + /^#ForwardToConsole=/ { $0 = "ForwardToConsole=yes"; ++f } + { print } END { assert(f == 1, "f == 1") }' /etc/systemd/journald.conf + + echo "-- Configuring ssh." + gawk -i assert -i inplace ' + /^#PasswordAuthentication / { $0 = "PasswordAuthentication no"; ++f1 } + /^#PermitRootLogin / { $0 = "PermitRootLogin no"; ++f2 } + { print } END { assert(f1 * f2 == 1, "f == 1") }' /etc/ssh/sshd_config + systemctl --quiet enable sshd + + echo "-- Configuring pacman." + curl --silent --show-error -o /etc/pacman.d/mirrorlist \ + 'https://www.archlinux.org/mirrorlist/?country=all&ip_version=4&use_mirror_status=on' + gawk -i assert -i inplace ' + /^#Server / { $0 = substr($0, 2); ++f } + { print } END { assert(f > 0, "f > 0") }' /etc/pacman.d/mirrorlist + cat <<-'EOF' > /etc/systemd/system/pacman-init.service + [Unit] + Description=Pacman keyring initialization + ConditionDirectoryNotEmpty=!/etc/pacman.d/gnupg + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/pacman-key --init + ExecStart=/usr/bin/pacman-key --populate archlinux + + [Install] + WantedBy=multi-user.target + EOF + systemctl --quiet enable pacman-init + + echo "-- Enabling other services." + systemctl --quiet enable \ + dhclient@ens4 rngd growpartfs@- \ + google-accounts-daemon google-clock-skew-daemon google-instance-setup \ + google-network-daemon google-shutdown-scripts google-startup-scripts + + echo "-- Configuring initcpio." + gawk -i assert -i inplace ' + /^MODULES=/ { $0 = "MODULES=(virtio_pci virtio_scsi sd_mod ext4)"; ++f1 } + /^BINARIES=/ { $0 = "BINARIES=(fsck fsck.ext4)"; ++f2 } + /^HOOKS=/ { $0 = "HOOKS=(base modconf)"; ++f3 } + { print } END { assert(f1 * f2 * f3 == 1, "f == 1") }' /etc/mkinitcpio.conf + gawk -i assert -i inplace ' + /^PRESETS=/ { $0 = "PRESETS=(default)"; ++f } + /#?fallback_/ { next } + { print } END { assert(f == 1, "f == 1") }' /etc/mkinitcpio.d/linux.preset + rm /boot/initramfs-linux-fallback.img + mkinitcpio --nocolor --preset linux + + echo "-- Configuring grub." + grub-install --target=i386-pc -- "$1" + cat <<-'EOF' > /etc/default/grub + # GRUB boot loader configuration + GRUB_CMDLINE_LINUX="console=ttyS0,38400n8 elevator=noop scsi_mod.use_blk_mq=Y" + GRUB_PRELOAD_MODULES="part_gpt part_msdos" + GRUB_TIMEOUT=0 + GRUB_DISABLE_RECOVERY=true + EOF + grub-mkconfig -o /boot/grub/grub.cfg + + # Workaround a bug where shutdown gets stuck on failed DNS resolution. + gawk -i assert -i inplace ' + /^hosts:/ && sub(/\s*resolve \[!UNAVAIL=return\]/, "") { ++f } + { print } END { assert(f == 1, "f == 1") }' /etc/nsswitch.conf + +EOS + +echo "- Cleaning up and finalizing the image." +> "$mount_dir/etc/machine-id" +> "$mount_dir/var/lib/dbus/machine-id" +rm -- "$mount_dir/var/log/pacman.log" +umount -- "$mount_dir" +unset mount_dir + +echo "- Building the compressed image." +disk_tar="arch-v$(date --utc +%Y%m%d).tar.gz" +tar --sparse -czf "$work_dir/$disk_tar" --directory="$work_dir" disk.raw +mv -- "$work_dir/$disk_tar" . + +echo "Successfully built image \`$disk_tar\`." diff --git a/build-arch-on-gce.sh b/build-arch-on-gce.sh deleted file mode 100755 index 3f136b0..0000000 --- a/build-arch-on-gce.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a Debian VM to build an Arch Linux image. - -INSTANCE_ID=${RANDOM} -INSTANCE_NAME=archbuilder${INSTANCE_ID} -ZONE_NAME=us-central1-f -MACHINE_TYPE=n1-standard-2 -GIT_SOURCE_URI=https://github.com/GoogleCloudPlatform/compute-archlinux-image-builder.git -SCRIPT_PARAMS="$*" - -function GcloudNotConfiguredHelp() { - echo "gcloud is missing or project is not set." - echo "Run these commands:" - echo " gcloud auth login" - echo " gcloud config set project <project>" - echo "" - echo "To install Cloud SDK go here: https://developers.google.com/cloud/sdk/" -} - -function MissingRequiredFlagsHelp() { - echo "--upload parameter is required to build on VM." - echo "Example: --upload gs://cloud-storage-bucket/archlinux.tar.gz" - echo "" - echo "Cloud Storage Buckets already in your project:" - gsutil ls gs:// -} - -function PrintHelp() { - ./build-gce-arch.py --help -} - -function DeployVm() { - echo "Creating Instance, ${INSTANCE_NAME}" - gcloud compute instances create ${INSTANCE_NAME} \ - --image ubuntu-14-10 \ - --machine-type ${MACHINE_TYPE} \ - --zone ${ZONE_NAME} \ - --metadata-from-file startup-script=gcevm-script-build-arch.sh \ - --metadata \ - script-params="${SCRIPT_PARAMS}" \ - instance-name="${INSTANCE_NAME}" \ - instance-zone="${ZONE_NAME}" \ - git-source-uri="${GIT_SOURCE_URI}" \ - --scopes compute-rw storage-full - echo "You can monitor progress of the build via:" - echo " gcloud compute instances get-serial-port-output ${INSTANCE_NAME} --zone ${ZONE_NAME} | grep startupscript" -} - - -GCLOUD_PROJECT_CONFIGURED=$(gcloud config list --all | grep project) -if [ "${GCLOUD_PROJECT_CONFIGURED}" != "" ]; then - if [[ "${SCRIPT_PARAMS}" == *--help* ]]; then - PrintHelp - elif [[ "${SCRIPT_PARAMS}" == *--upload* ]]; then - echo "Creating VM to build Arch Linux" - DeployVm - else - MissingRequiredFlagsHelp - fi -else - GcloudNotConfiguredHelp -fi diff --git a/build-gce-arch.py b/build-gce-arch.py deleted file mode 100755 index cc80800..0000000 --- a/build-gce-arch.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import os -import logging -import sys -from datetime import date - -import utils - -COMPUTE_IMAGE_PACKAGES_GIT_URL = ( - 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') -IMAGE_FILE='disk.raw' -SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() -SETUP_PACKAGES = ('pacman wget gcc make parted git setconf libaio sudo ' - 'fakeroot arch-install-scripts').split() -IMAGE_PACKAGES = ('base tar wget ' - 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' - 'ntp psmisc openssh udev less bash-completion zip unzip ' - 'python2 python3').split() - - -def main(): - args = ParseArgs() - utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) - logging.info('Arch Linux Image Builder') - logging.info('========================') - - workspace_dir = None - image_file = None - try: - aur_packages = InstallPackagesOnHostMachine() - image_path = CreateArchImage(args, aur_packages) - image_name, image_filename, image_description = GetImageNameAndDescription( - args.outfile) - image_file = SaveImage(image_path, image_filename) - if args.upload and image_file: - UploadImage(image_file, args.upload, make_public=args.public) - if args.register: - AddImageToComputeEngineProject( - image_name, args.upload, image_description) - finally: - if not args.nocleanup and workspace_dir: - utils.DeleteDirectory(workspace_dir) - - -def CreateArchImage(args, aur_packages): - image_path = os.path.join(os.getcwd(), IMAGE_FILE) - CreateBlankImage(image_path, size_gb=int(args.size_gb), fs_type=args.fs_type) - mount_path = utils.CreateTempDirectory(base_dir='/') - image_mapping = utils.ImageMapper(image_path, mount_path) - try: - image_mapping.InstallLoopback() - image_mapping.Map() - primary_mapping = image_mapping.GetFirstMapping() - image_mapping_path = primary_mapping['path'] - FormatImage(image_mapping_path) - try: - image_mapping.Mount() - utils.CreateDirectory('/run/shm') - utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) - InstallArchLinux(mount_path) - disk_uuid = SetupFileSystem(mount_path, image_mapping_path, args.fs_type) - ConfigureArchInstall( - args, mount_path, primary_mapping['parent'], disk_uuid, aur_packages) - utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) - PurgeDisk(mount_path) - finally: - image_mapping.Unmount() - ShrinkDisk(image_mapping_path) - finally: - image_mapping.Unmap() - utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) - utils.Sync() - return image_path - - -def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid, aur_packages): - relative_builder_path = utils.CopyBuilder(mount_path) - packages_dir = utils.CreateTempDirectory(mount_path) - utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) - utils.CreateDirectory(os.path.join(mount_path, '')) - aur_packages_dir = os.path.join(packages_dir, 'aur') - for aur_package in aur_packages: - utils.CopyFiles(aur_package, aur_packages_dir + '/') - packages_dir = os.path.relpath(packages_dir, mount_path) - params = { - 'packages_dir': '/%s' % packages_dir, - 'device': parent_path, - 'disk_uuid': disk_uuid, - 'accounts': args.accounts, - 'debugmode': args.debug, - 'quiet': args.quiet, - 'verbose': args.verbose, - 'packages': args.packages, - 'size_gb': args.size_gb - } - config_arch_py = os.path.join( - '/', relative_builder_path, 'arch-image.py') - utils.RunChroot(mount_path, - '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), - use_custom_path=False) - utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) - - -def InstallPackagesOnHostMachine(): - aur_packages = [] - utils.UpdatePacmanDatabase() - utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) - utils.InstallPackages(SETUP_PACKAGES) - utils.UpdateAllPackages() - aur_packages.append(utils.AurInstall(name='multipath-tools-git')) - aur_packages.append(utils.AurInstall(name='zerofree')) - aur_packages.append(utils.AurInstall(name='python2-crcmod')) - return aur_packages - - -def CreateBlankImage(image_path, size_gb=10, fs_type='ext4'): - utils.LogStep('Create Image') - utils.Run(['rm', '-f', image_path]) - utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) - utils.Run(['parted', image_path, 'mklabel', 'msdos']) - utils.Run(['parted', image_path, 'mkpart', 'primary', - fs_type, '1', str(int(size_gb) * 1024)]) - - -def FormatImage(image_mapping_path): - utils.LogStep('Format Image') - utils.Run(['mkfs', image_mapping_path]) - utils.Sync() - - -def InstallArchLinux(base_dir): - utils.LogStep('Install Arch Linux') - utils.Pacstrap(base_dir, IMAGE_PACKAGES) - - -def SetupFileSystem(base_dir, image_mapping_path, fs_type): - utils.LogStep('File Systems') - _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], - capture_output=True) - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) - _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', - '-o', 'value', - image_mapping_path], - capture_output=True) - disk_uuid = disk_uuid.strip() - utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), - 'UUID=%s / %s defaults 0 1' % (disk_uuid, fs_type)) - utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) - return disk_uuid - - -def PurgeDisk(mount_path): - paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] - for path in paths: - utils.DeleteDirectory(os.path.join(mount_path, path)) - - -def ShrinkDisk(image_mapping_path): - utils.LogStep('Shrink Disk') - utils.Run(['zerofree', image_mapping_path]) - - -def SaveImage(disk_image_file, image_filename): - utils.LogStep('Save Arch Linux Image in GCE format') - image_raw = os.path.join(os.getcwd(), IMAGE_FILE) - gce_image_file = os.path.join(os.getcwd(), image_filename) - utils.Run(['tar', '-Szcf', image_filename, IMAGE_FILE]) - return gce_image_file - - -def UploadImage(image_path, gs_path, make_public=False): - utils.LogStep('Upload Image to Cloud Storage') - utils.SecureDeleteFile('~/.gsutil/*.url') - utils.Run(['gsutil', 'rm', gs_path], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - utils.Run(['gsutil', 'cp', image_path, gs_path], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - if make_public: - utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - - -def AddImageToComputeEngineProject(image_name, gs_path, description): - utils.LogStep('Add image to project') - utils.Run( - ['gcloud', 'compute', 'images', 'delete', image_name, '-q'], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - utils.Run( - ['gcloud', 'compute', 'images', 'create', image_name, '-q', - '--source-uri', gs_path, - '--description', description], - env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) - - -def GetImageNameAndDescription(outfile_name): - today = date.today() - isodate = today.strftime("%Y-%m-%d") - yyyymmdd = today.strftime("%Y%m%d") - image_name = 'arch-v%s' % yyyymmdd - if outfile_name: - image_filename = outfile_name - else: - image_filename = '%s.tar.gz' % image_name - description = 'Arch Linux x86-64 built on %s' % isodate - return image_name, image_filename, description - - -def ParseArgs(): - parser = argparse.ArgumentParser( - description='Arch Linux Image Builder for Compute Engine') - parser.add_argument('-p', '--packages', - dest='packages', - nargs='+', - help='Additional packages to install via Pacman.') - parser.add_argument('-v', '--verbose', - dest='verbose', - default=False, - help='Verbose console output.', - action='store_true') - parser.add_argument('-q', '--quiet', - dest='quiet', - default=False, - help='Suppress all console output.', - action='store_true') - parser.add_argument('--upload', - dest='upload', - default=None, - help='Google Cloud Storage path to upload to.') - parser.add_argument('--size_gb', - dest='size_gb', - default=10, - help='Volume size of image (in GiB).') - parser.add_argument('--accounts', - dest='accounts', - nargs='+', - help='Space delimited list of user accounts to create on ' - 'the image. Format: username:password') - parser.add_argument('--nocleanup', - dest='nocleanup', - default=False, - help='Prevent cleaning up the image build workspace ' - 'after image has been created.', - action='store_true') - parser.add_argument('--outfile', - dest='outfile', - default=None, - help='Name of the output image file.') - parser.add_argument('--debug', - dest='debug', - default=False, - help='Configure the image for debugging.', - action='store_true') - parser.add_argument('--public', - dest='public', - default=False, - help='Make image file uploaded to Cloud Storage ' - 'available for everyone.', - action='store_true') - parser.add_argument('--register', - dest='register', - default=False, - help='Add the image to Compute Engine project. ' - '(Upload to Cloud Storage required.)', - action='store_true') - parser.add_argument('--nopacmankeys', - dest='nopacmankeys', - default=False, - help='Disables signature checking for pacman packages.', - action='store_true') - parser.add_argument('--fs_type', - dest='fs_type', - default='ext4', - help='Verbose console output.', - action='store_true') - return parser.parse_args() - - -if __name__ == '__main__': - main() diff --git a/gcevm-script-build-arch.sh b/gcevm-script-build-arch.sh deleted file mode 100755 index 8ddaf67..0000000 --- a/gcevm-script-build-arch.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build an Arch Linux image from within a GCE Debian VM. - -BUILDER_ROOT=/mnt/archongce/source -INSTANCE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-name) -ZONE_NAME=$(/usr/share/google/get_metadata_value attributes/instance-zone) -SCRIPT_PARAMS=$(/usr/share/google/get_metadata_value attributes/script-params) -SCRIPT_PARAMS="--verbose --register ${SCRIPT_PARAMS}" -GIT_SOURCE_URI=$(/usr/share/google/get_metadata_value attributes/git-source-uri) -REMOTE_IMAGE=$(echo "i = '${SCRIPT_PARAMS}'.split(); print i[i.index('--upload') + 1]" | python) - -echo "Builder Root: ${BUILDER_ROOT}" -echo "Instance Name: ${INSTANCE_NAME}" -echo "Instance Zone: ${ZONE_NAME}" -echo "Source Git Repository: ${GIT_SOURCE_URI}" -echo "Remote Image: ${REMOTE_IMAGE}" -echo "Running script as:" -echo " ./build-gce-arch.py ${SCRIPT_PARAMS}" - -echo "Setup Builder Environment" -mkdir -p ${BUILDER_ROOT} - -echo "Updating Cloud SDK" -yes | gcloud components update - -function InstallDependenciesForDebian { - echo "Installing Dependencies (Debian)" - apt-get update - apt-get install -y -qq python3 haveged git -} - -function InstallDependenciesForRedhat { - echo "Installing Dependencies (Redhat)" - rpm -Uvh http://download.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-5.noarch.rpm - yum install -y python3 haveged git -} -if [ -f /etc/redhat-release ] -then - InstallDependenciesForRedhat -else - InstallDependenciesForDebian -fi - -echo "Getting source code..." -git clone ${GIT_SOURCE_URI} ${BUILDER_ROOT} - -cd ${BUILDER_ROOT} -haveged -w 1024 -gsutil rm ${REMOTE_IMAGE} -./build-gce-arch.py ${SCRIPT_PARAMS} - -function SaveLogForRedhat { - journalctl > builder.log -} - -function SaveLogForDebian { - cat /var/log/syslog | grep -o "startupscript.*" > builder.log -} - -if [ -f /etc/redhat-release ] -then - SaveLogForRedhat -else - SaveLogForDebian -fi - -gsutil cp builder.log ${REMOTE_IMAGE}.log -gcloud compute -q instances delete ${INSTANCE_NAME} --zone ${ZONE_NAME} diff --git a/push.sh b/push.sh deleted file mode 100755 index 1441a29..0000000 --- a/push.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -VM_USER="${USER}" -PACKAGE_FILE="archbuilder.tar.gz" -INSTANCE_NAME="instance-1" -ZONE="us-east1-d" -ARCH_DATE="20151203" -SSH_TARGET=${VM_USER}@${INSTANCE_NAME} - -rm -f ${PACKAGE_FILE} -tar czf ${PACKAGE_FILE} * -gcloud compute ssh ${SSH_TARGET} --command "rm -fr *" --zone=${ZONE} -gcloud compute copy-files ${PACKAGE_FILE} ${SSH_TARGET}:/home/${VM_USER} --zone=${ZONE} - -gcloud compute ssh ${SSH_TARGET} --command "tar xvzf ${PACKAGE_FILE}; rm ${PACKAGE_FILE}; chmod +x *.sh" --zone=${ZONE} -gcloud compute ssh ${SSH_TARGET} --command "sudo ./build-gce-arch.py --verbose --size_gb=100 --debug --public --upload gs://gce-arch-images/unverified/arch-v${ARCH_DATE}.tar.gz --register" --zone=${ZONE} diff --git a/utils.py b/utils.py deleted file mode 100644 index 3ed8082..0000000 --- a/utils.py +++ /dev/null @@ -1,388 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import glob -import gzip -import hashlib -import json -import os -import logging -import pwd -import shutil -import subprocess -import sys -import tempfile -import time -import urllib.request, urllib.error, urllib.parse - - -APP_NAME = 'archbuilder' -BUILDER_USER = 'nobody' -ETC_LOCALE_GEN = ''' -en_US.UTF-8 UTF-8 -en_US ISO-8859-1 -''' - - -def LogStep(msg): - logging.info('- %s', msg) - - -def SetupLogging(quiet=False, verbose=False): - if not quiet: - root = logging.getLogger() - stdout_logger = logging.StreamHandler(sys.stdout) - if verbose: - stdout_logger.setLevel(logging.DEBUG) - else: - stdout_logger.setLevel(logging.WARNING) - stdout_logger.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) - root.addHandler(stdout_logger) - root.setLevel(logging.DEBUG) - - -def Sed(file_path, modification): - Run(['sed', '-i', modification, file_path]) - - -def Replace(file_path, pattern, replacement): - Sed(file_path, 's/%s/%s/g' % (pattern, replacement)) - - -def ReplaceLine(file_path, pattern, replacement): - Sed(file_path, '/%s/c\%s' % (pattern, replacement)) - - -def SudoRun(params, cwd=None, capture_output=False): - if os.geteuid() != 0: - params = ['sudo'] + params - return Run(params, capture_output=capture_output) - - -def UserExists(username): - try: - pwd.getpwnam(username) - return True - except: - return False - - -def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): - try: - logging.debug('Run: %s in %s', params, cwd) - if env: - new_env = os.environ.copy() - new_env.update(env) - env = new_env - out_pipe = None - if capture_output: - out_pipe = subprocess.PIPE - proc = subprocess.Popen(params, stdout=out_pipe, cwd=cwd, shell=shell, env=env) - if not wait: - return 0, '', '' - out, err = proc.communicate() - if capture_output: - logging.debug(out) - - if out: - out = out.decode(encoding='UTF-8') - if err: - err = err.decode(encoding='UTF-8') - except KeyboardInterrupt: - return 1, '', '' - except: - logging.error(sys.exc_info()[0]) - logging.error(sys.exc_info()) - Run(['/bin/bash']) - return 1, '', '' - - return proc.returncode, out, err - - -def DownloadFile(url, file_path): - Run(['wget', '-O', file_path, url, '-nv']) - - -def HttpGet(url): - return str(urllib.request.urlopen(url).read(), encoding='utf8') - - -def Sha1Sum(file_path): - with open(file_path, 'rb') as fp: - return hashlib.sha1(fp.read()).hexdigest() - - -def Untar(file_path, dest_dir): - Run(['tar', '-C', dest_dir, '-xzf', file_path]) - - -def Chmod(fp, mode): - Run(['chmod', str(mode), fp]) - - -def CreateDirectory(dir_path): - dir_path = ToAbsolute(dir_path) - if not os.path.isdir(dir_path): - os.makedirs(dir_path) - - -def ToAbsolute(path): - if not path: - return path - return os.path.expandvars(os.path.expanduser(path)) - - -def DeleteFile(file_pattern): - DeleteFileFunc(file_pattern, lambda item: os.remove(item)) - - -def SecureDeleteFile(file_pattern): - DeleteFileFunc(file_pattern, lambda item: Run(['shred', '--remove', '--zero', item])) - - -def DeleteFileFunc(file_pattern, delete_func): - items = glob.glob(ToAbsolute(file_pattern)) - for item in items: - logging.warning('Deleting %s', item) - delete_func(item) - - -def DirectoryExists(dir_path): - return os.path.exists(dir_path) - - -def DeleteDirectory(dir_path): - if DirectoryExists(dir_path): - shutil.rmtree(dir_path) - - -def CreateTempDirectory(base_dir=None): - return tempfile.mkdtemp(dir=ToAbsolute(base_dir), prefix='gcearch') - - -def WriteFile(path, content): - with open(ToAbsolute(path), 'w') as fp: - fp.write(content) - - -def AppendFile(path, content): - with open(ToAbsolute(path), 'a') as fp: - fp.write(content) - - -def RunChroot(base_dir, command, use_custom_path=True): - base_dir = ToAbsolute(base_dir) - if use_custom_path: - chroot_file = os.path.join(base_dir, 'bin/arch-chroot') - else: - chroot_file = 'arch-chroot' - SudoRun([chroot_file, base_dir, '/bin/bash', '-c', command]) - - -def CopyFiles(source_pattern, dest): - """Copies a set of files based on glob pattern to a directory. - Avoiding shutil.copyfile because of bugs.python.org/issue10016.""" - items = glob.glob(ToAbsolute(source_pattern)) - for item in items: - Run(['cp', '-Rf', item, dest]) - - -def CopyBuilder(base_dir): - script_dir = os.path.dirname(os.path.realpath(__file__)) - temp_dir = CreateTempDirectory(base_dir=base_dir) - DeleteDirectory(temp_dir) - relative_dir = os.path.relpath(temp_dir, base_dir) - shutil.copytree(script_dir, temp_dir, ignore=shutil.ignore_patterns('*.tar.gz', '*.raw')) - return relative_dir - - -def EncodeArgs(decoded_args): - return base64.standard_b64encode(gzip.compress(bytes(json.dumps(decoded_args), 'utf-8'))).decode('utf-8') - - -def DecodeArgs(encoded_args): - return json.loads(gzip.decompress(base64.standard_b64decode(encoded_args)).decode('utf-8')) - - -def DebugBash(): - Run(['/bin/bash']) - - -def DebugPrintFile(file_path): - logging.info('==============================================================') - logging.info('File: %s', file_path) - logging.info('==============================================================') - Run(['cat', file_path]) - - -def Sync(): - Run(['sync']) - - -def EnableService(service_name): - Run(['systemctl', 'enable', service_name]) - - -def DisableService(service_name): - Run(['systemctl', 'disable', service_name]) - - -def Symlink(source_file, dest_file): - Run(['ln', '-s', source_file, dest_file]) - - -def ChangeDirectoryOwner(username, directory): - SudoRun(['chown', '-R', username, directory]) - - -def AurInstall(name=None, pkbuild_url=None): - if name: - pkbuild_url = 'https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD?h=%s' % (name.lower()) - workspace_dir = CreateTempDirectory() - DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) - ChangeDirectoryOwner(BUILDER_USER, workspace_dir) - Run(['runuser', '-m', BUILDER_USER, '-c', 'makepkg'], cwd=workspace_dir) - tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) - tarball = tarball[0] - Pacman(['-U', tarball], cwd=workspace_dir) - - return tarball - - -def Pacstrap(base_dir, params): - Run(['pacstrap', base_dir] + params) - - -def Pacman(params, cwd=None): - Run(['pacman', '--noconfirm'] + params, cwd=cwd) - - -def UpdatePacmanDatabase(): - Pacman(['-Sy']) - - -def UpdateAllPackages(): - Pacman(['-Syyu']) - -def InstallPackages(package_list): - Pacman(['-S'] + package_list) - - -def SetupArchLocale(): - AppendFile('/etc/locale.gen', ETC_LOCALE_GEN) - Run(['locale-gen']) - Run(['localectl', 'set-locale', 'LANG="en_US.UTF-8"', 'LC_COLLATE="C"']) - - -class ImageMapper(object): - """Interface for kpartx, mount, and umount.""" - def __init__(self, raw_disk, mount_path): - self._raw_disk = raw_disk - self._mount_path = mount_path - self._device_map = None - self._mount_points = None - - def _LoadPartitionsIfNeeded(self): - if not self._device_map: - self.LoadPartitions() - - def InstallLoopback(self): - SudoRun(['modprobe', 'loop']) - - def LoadPartitions(self): - return_code, out, err = SudoRun(['kpartx', '-l', self._raw_disk], capture_output=True) - # Expected Format - # loop2p1 : 0 10483712 /dev/loop2 2048 - # loop2p2 : 0 1 /dev/loop2 2047 - # loop deleted : /dev/loop2 - - self._device_map = {} - lines = out.splitlines() - for line in lines: - parts = str(line).split() - if len(parts) == 6: - mapping = { - 'name': parts[0], - 'size_blocks': parts[3], - 'parent': parts[4], - 'start_block': parts[5], - 'path': '/dev/mapper/%s' % str(parts[0]) - } - logging.info('Mapping: %s', mapping) - self._device_map[mapping['name']] = mapping - if len(self._device_map) == 1: - self._mount_points = [self._mount_path] - else: - self._mount_points = [] - for name in list(self._device_map.keys()): - self._mount_points.append(os.path.join(self._mount_path, name)) - - def ForEachDevice(self, func): - for name in list(self._device_map.keys()): - spec = self._device_map[name] - func(spec) - - def ForEachDeviceWithIndex(self, func): - i = 0 - for name in list(self._device_map.keys()): - spec = self._device_map[name] - func(i, spec) - i += 1 - - def GetFirstMapping(self): - logging.info('DeviceMap: %s', self.GetDeviceMap()) - return next(iter(self.GetDeviceMap().values())) - - def GetDeviceMap(self): - return self._device_map - - def Sync(self): - Run(['sync']) - - def Map(self): - SudoRun(['kpartx', '-a', '-v', '-s', self._raw_disk]) - self.LoadPartitions() - - def Unmap(self): - self.Sync() - time.sleep(2) - SudoRun(['kpartx', '-d', '-v', '-s', self._raw_disk]) - self._device_map = None - - def Mount(self): - self._LoadPartitionsIfNeeded() - self._ThrowIfBadMountMap() - def MountCmd(index, spec): - mount_point = self._mount_points[index] - CreateDirectory(mount_point) - SudoRun(['mount', spec['path'], mount_point]) - self.ForEachDeviceWithIndex(MountCmd) - - def Unmount(self): - self._LoadPartitionsIfNeeded() - self._ThrowIfBadMountMap() - for path in self._mount_points: - SudoRun(['umount', path]) - self.Sync() - time.sleep(2) - - def _ThrowIfBadMountMap(self): - if not self._mount_points: - raise IOError('Attempted to found {0} without a mount points.'.format(self._raw_disk)) - if len(self._mount_points) != len(list(self._device_map.keys())): - raise IOError('Number of device maps ({0}) does not match mount points ({1}).'.format( - len(list(self._device_map.keys())), len(self._mount_points) - )) |
