aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJeremy Edwards <jeremyedwards@google.com>2014-08-08 10:52:30 -0700
committerJeremy Edwards <jeremyedwards@google.com>2014-08-08 10:52:30 -0700
commit9fab134a7408c1d6edb46b7735e95f0d03e1a3e6 (patch)
treed8ff2945bb821787c353bf3e9c3289179d81ea19
downloadcompute-archlinux-image-builder-9fab134a7408c1d6edb46b7735e95f0d03e1a3e6.tar.xz
Initial version.
-rw-r--r--.gitignore6
-rw-r--r--CONTRIB.md64
-rw-r--r--LICENSE203
-rw-r--r--README.md60
-rwxr-xr-xarch-image.py521
-rwxr-xr-xarch-staging.py147
-rwxr-xr-xbuild-gce-arch.py244
-rw-r--r--utils.py354
8 files changed, 1599 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..951b277
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+# Files to ignore
+*.log
+*.tar.gz
+*.pyc
+*.raw
+__pycache__/*
diff --git a/CONTRIB.md b/CONTRIB.md
new file mode 100644
index 0000000..b368291
--- /dev/null
+++ b/CONTRIB.md
@@ -0,0 +1,64 @@
+# How to become a contributor and submit your own code
+
+## Contributor License Agreements
+
+We'd love to accept your sample apps and patches! Before we can take them, we
+have to jump a couple of legal hurdles.
+
+Please fill out either the individual or corporate Contributor License Agreement
+(CLA).
+
+ * If you are an individual writing original source code and you're sure you
+ own the intellectual property, then you'll need to sign an [individual CLA]
+ (https://developers.google.com/open-source/cla/individual).
+ * If you work for a company that wants to allow you to contribute your work,
+ then you'll need to sign a [corporate CLA]
+ (https://developers.google.com/open-source/cla/corporate).
+
+Follow either of the two links above to access the appropriate CLA and
+instructions for how to sign and return it. Once we receive it, we'll be able to
+accept your pull requests.
+
+## Contributing A Patch
+
+1. Submit an issue describing your proposed change to the repo in question.
+1. The repo owner will respond to your issue promptly.
+1. If your proposed change is accepted, and you haven't already done so, sign a
+ Contributor License Agreement (see details above).
+1. Fork the desired repo, develop and test your code changes.
+1. Ensure that your code adheres to the existing style in the sample to which
+ you are contributing. Refer to the
+ [Google Cloud Platform Samples Style Guide]
+ (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the
+ recommended coding standards for this organization.
+1. Ensure that your code has an appropriate set of unit tests which all pass.
+1. Submit a pull request.
+
+## Contributing A New Sample App
+
+1. Submit an issue to the GoogleCloudPlatform/Template repo describing your
+ proposed sample app.
+1. The Template repo owner will respond to your enhancement issue promptly.
+ Instructional value is the top priority when evaluating new app proposals for
+ this collection of repos.
+1. If your proposal is accepted, and you haven't already done so, sign a
+ Contributor License Agreement (see details above).
+1. Create your own repo for your app following this naming convention:
+ * {product}-{app-name}-{language}
+ * products: appengine, compute, storage, bigquery, prediction, cloudsql
+ * example: appengine-guestbook-python
+ * For multi-product apps, concatenate the primary products, like this:
+ compute-appengine-demo-suite-python.
+ * For multi-language apps, concatenate the primary languages like this:
+ appengine-sockets-python-java-go.
+
+1. Clone the README.md, CONTRIB.md and LICENSE files from the
+ GoogleCloudPlatform/Template repo.
+1. Ensure that your code adheres to the existing style in the sample to which
+ you are contributing. Refer to the
+ [Google Cloud Platform Samples Style Guide]
+ (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the
+ recommended coding standards for this organization.
+1. Ensure that your code has an appropriate set of unit tests which all pass.
+1. Submit a request to fork your repo in GoogleCloudPlatform organizationt via
+ your proposal issue. \ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..15f9dd5
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,203 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2014 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ \ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f3f628e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,60 @@
+## Arch Linux Image Builder for GCE
+
+Creates an Arch Linux image that can run on Google Compute Engine.
+
+The image is configured close to the recommendations listed on [Building an image from scratch](https://developers.google.com/compute/docs/images#buildingimage).
+
+These scripts are written in Python3.
+
+## Usage
+
+### Install and Configure Cloud SDK (one time setup)
+```
+# Install Cloud SDK (https://developers.google.com/cloud/sdk/)
+# For linux:
+curl https://sdk.cloud.google.com | bash
+
+gcloud auth login
+gcloud config set project <project>
+# Your project ID in Cloud Console, https://console.developers.google.com/
+```
+
+### On a Compute Engine VM (recommended)
+```
+./build-arch-on-gce.sh --upload gs://${BUCKET}/archlinux.tar.gz
+
+# You will need a Cloud Storage bucket.
+# List buckets owned by your project.
+gsutil ls gs://
+# Create a new bucket
+gsutil mb gs://${BUCKET}
+```
+
+### Locally
+```
+# Install Required Packages
+# Arch Linux
+sudo pacman -S python haveged
+# Debian
+sudo apt-get -y install python3 haveged
+# Redhat
+sudo yum install -y python3 haveged
+
+./build-gce-arch.py --verbose
+# Upload to Cloud Storage
+gsutil cp archlinux-gce.tar.gz gs://${BUCKET}/archlinux.tar.gz
+
+# Add image to project
+gcloud compute images insert archlinux \
+ --source-uri gs://${BUCKET}/archlinux.tar.gz \
+ --description "Arch Linux for Compute Engine"
+```
+
+
+## Contributing changes
+
+* See [CONTRIB.md](CONTRIB.md)
+
+
+## Licensing
+All files in this repository are under the [Apache License, Version 2.0](LICENSE) unless noted otherwise.
diff --git a/arch-image.py b/arch-image.py
new file mode 100755
index 0000000..04bf6e4
--- /dev/null
+++ b/arch-image.py
@@ -0,0 +1,521 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import sys
+
+import utils
+
+
+ETC_MOTD = '''Arch Linux for Compute Engine
+'''
+
+ETC_HOSTS = '''127.0.0.1 localhost
+169.254.169.254 metadata.google.internal metadata
+'''
+
+ETC_SSH_SSH_CONFIG = '''
+Host *
+Protocol 2
+ForwardAgent no
+ForwardX11 no
+HostbasedAuthentication no
+StrictHostKeyChecking no
+Ciphers aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc
+Tunnel no
+
+# Google Compute Engine times out connections after 10 minutes of inactivity.
+# Keep alive ssh connections by sending a packet every 7 minutes.
+ServerAliveInterval 420
+'''
+
+ETC_SSH_SSHD_CONFIG = '''
+# Disable PasswordAuthentication as ssh keys are more secure.
+PasswordAuthentication no
+
+# Disable root login, using sudo provides better auditing.
+PermitRootLogin no
+
+PermitTunnel no
+AllowTcpForwarding yes
+X11Forwarding no
+
+# Compute times out connections after 10 minutes of inactivity. Keep alive
+# ssh connections by sending a packet every 7 minutes.
+ClientAliveInterval 420
+
+# Restrict sshd to just IPv4 for now as sshd gets confused for things
+# like X11 forwarding.
+
+Port 22
+Protocol 2
+
+UsePrivilegeSeparation yes
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 768
+
+SyslogFacility AUTH
+LogLevel INFO
+
+LoginGraceTime 120
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+
+IgnoreRhosts yes
+RhostsRSAAuthentication no
+HostbasedAuthentication no
+
+PermitEmptyPasswords no
+ChallengeResponseAuthentication no
+
+PasswordAuthentication no
+PrintMotd no
+PrintLastLog yes
+
+TCPKeepAlive yes
+
+Subsystem sftp /usr/lib/openssh/sftp-server
+
+UsePAM yes
+UseDNS no
+'''
+
+ETC_SYSCTL_D_70_DISABLE_IPV6_CONF = '''
+net.ipv6.conf.all.disable_ipv6 = 1
+'''
+
+ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF = '''
+# enables syn flood protection
+net.ipv4.tcp_syncookies = 1
+
+# ignores source-routed packets
+net.ipv4.conf.all.accept_source_route = 0
+
+# ignores source-routed packets
+net.ipv4.conf.default.accept_source_route = 0
+
+# ignores ICMP redirects
+net.ipv4.conf.all.accept_redirects = 0
+
+# ignores ICMP redirects
+net.ipv4.conf.default.accept_redirects = 0
+
+# ignores ICMP redirects from non-GW hosts
+net.ipv4.conf.all.secure_redirects = 1
+
+# ignores ICMP redirects from non-GW hosts
+net.ipv4.conf.default.secure_redirects = 1
+
+# don't allow traffic between networks or act as a router
+net.ipv4.ip_forward = 0
+
+# don't allow traffic between networks or act as a router
+net.ipv4.conf.all.send_redirects = 0
+
+# don't allow traffic between networks or act as a router
+net.ipv4.conf.default.send_redirects = 0
+
+# reverse path filtering - IP spoofing protection
+net.ipv4.conf.all.rp_filter = 1
+
+# reverse path filtering - IP spoofing protection
+net.ipv4.conf.default.rp_filter = 1
+
+# reverse path filtering - IP spoofing protection
+net.ipv4.conf.default.rp_filter = 1
+
+# ignores ICMP broadcasts to avoid participating in Smurf attacks
+net.ipv4.icmp_echo_ignore_broadcasts = 1
+
+# ignores bad ICMP errors
+net.ipv4.icmp_ignore_bogus_error_responses = 1
+
+# logs spoofed, source-routed, and redirect packets
+net.ipv4.conf.all.log_martians = 1
+
+# log spoofed, source-routed, and redirect packets
+net.ipv4.conf.default.log_martians = 1
+
+# implements RFC 1337 fix
+net.ipv4.tcp_rfc1337 = 1
+
+# randomizes addresses of mmap base, heap, stack and VDSO page
+kernel.randomize_va_space = 2
+'''
+
+ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF = '''
+# provides protection from ToCToU races
+fs.protected_hardlinks=1
+
+# provides protection from ToCToU races
+fs.protected_symlinks=1
+
+# makes locating kernel addresses more difficult
+kernel.kptr_restrict=1
+
+# set ptrace protections
+kernel.yama.ptrace_scope=1
+
+# set perf only available to root
+kernel.perf_event_paranoid=2
+'''
+
+ETC_PAM_D_PASSWD = '''
+#%PAM-1.0
+password required pam_cracklib.so difok=2 minlen=8 dcredit=2 ocredit=2 retry=3
+password required pam_unix.so sha512 shadow nullok
+password required pam_tally.so even_deny_root_account deny=3 lock_time=5 unlock_time=3600
+'''
+
+ETC_SUDOERS_D_ADD_GROUP_ADM = '''
+%adm ALL=(ALL) ALL
+'''
+
+ETC_FAIL2BAN_JAIL_LOCAL = '''
+[DEFAULT]
+backend = systemd
+loglevel = WARNING
+'''
+
+ETC_FAIL2BAN_JAIL_D_SSHD_CONF = '''
+# fail2ban SSH
+# block ssh after 3 unsuccessful login attempts for 10 minutes
+[sshd]
+enabled = true
+action = iptables[chain=INPUT, protocol=tcp, port=22, name=sshd]
+maxRetry = 3
+findtime = 600
+bantime = 600
+port = 22
+'''
+
+GCIMAGEBUNDLE_ARCH_PY = '''
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Arch Linux specific platform info."""
+
+
+import os
+
+from gcimagebundlelib import linux
+
+
+class Arch(linux.LinuxPlatform):
+ """Arch Linux specific information."""
+
+ @staticmethod
+ def IsThisPlatform(root='/'):
+ return os.path.isfile('/etc/arch-release')
+
+ def __init__(self):
+ super(Arch, self).__init__()
+'''
+
+
+def main():
+ args = utils.DecodeArgs(sys.argv[1])
+ utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose'])
+ logging.info('Setup Bootstrapper Environment')
+ SetupLocale()
+ ConfigureTimeZone()
+ ConfigureKernel()
+ InstallBootloader(args['device'], args['disk_uuid'], args['debugmode'])
+ ForwardSystemdToConsole()
+ SetupNtpServer()
+ SetupNetwork()
+ SetupSsh()
+ #SetupFail2ban()
+ SetupAccounts(args)
+ InstallGcePackages(args['packages_dir'])
+ ConfigMessageOfTheDay()
+ ConfigureSecurity()
+ ConfigureSerialPortOutput()
+ OptimizePackages()
+
+
+def SetupAccounts(args):
+ accounts = args['accounts']
+ if accounts:
+ utils.LogStep('Add Accounts')
+ for account in accounts:
+ username, password = account.split(':')
+ logging.info(' - %s', username)
+ utils.Run(['useradd', username, '-m', '-s', '/bin/bash',
+ '-G', 'adm,video'])
+ utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True)
+
+
+def OptimizePackages():
+ utils.LogStep('Cleanup Cached Package Data')
+ utils.Pacman(['-Syu'])
+ utils.Pacman(['-Sc'])
+ utils.Run(['pacman-optimize'])
+
+
+def SetupLocale():
+ utils.LogStep('Set Locale to US English (UTF-8)')
+ utils.SetupArchLocale()
+
+
+def ConfigureTimeZone():
+ utils.LogStep('Set Timezone to UTC')
+ utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime'])
+
+
+def ConfigureKernel():
+ utils.LogStep('Configure Kernel')
+ utils.Replace('/etc/mkinitcpio.conf',
+ 'MODULES=""',
+ 'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"')
+ utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '')
+ utils.Run(['mkinitcpio',
+ '-g', '/boot/initramfs-linux.img',
+ '-k', '/boot/vmlinuz-linux',
+ '-c', '/etc/mkinitcpio.conf'])
+
+
+def InstallBootloader(device, uuid, debugmode):
+ utils.LogStep('Install Syslinux bootloader')
+ '''
+ utils.Run(['syslinux-install_update', '-i', '-a', '-m'])
+ '''
+ utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device])
+ utils.CreateDirectory('/boot/syslinux')
+ utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/')
+ utils.Run(['extlinux', '--install', '/boot/syslinux'])
+ utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1')
+ utils.Run(['fdisk', '-l', device])
+ utils.Run(['dd', 'bs=440', 'count=1', 'conv=notrunc',
+ 'if=/usr/lib/syslinux/bios/mbr.bin', 'of=%s' % device])
+
+ boot_params = [
+ 'console=ttyS0,38400',
+ 'CONFIG_KVM_GUEST=y',
+ 'CONFIG_KVM_CLOCK=y',
+ 'CONFIG_VIRTIO_PCI=y',
+ 'CONFIG_SCSI_VIRTIO=y',
+ 'CONFIG_VIRTIO_NET=y',
+ 'CONFIG_STRICT_DEVMEM=y',
+ 'CONFIG_DEVKMEM=n',
+ 'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536',
+ 'CONFIG_DEBUG_RODATA=y',
+ 'CONFIG_DEBUG_SET_MODULE_RONX=y',
+ 'CONFIG_CC_STACKPROTECTOR=y',
+ 'CONFIG_COMPAT_VDSO=n',
+ 'CONFIG_COMPAT_BRK=n',
+ 'CONFIG_X86_PAE=y',
+ 'CONFIG_SYN_COOKIES=y',
+ 'CONFIG_SECURITY_YAMA=y',
+ 'CONFIG_SECURITY_YAMA_STACKED=y',
+ ]
+ if debugmode:
+ boot_params += [
+ 'systemd.log_level=debug',
+ 'systemd.log_target=console',
+ 'systemd.journald.forward_to_syslog=yes',
+ 'systemd.journald.forward_to_kmsg=yes',
+ 'systemd.journald.forward_to_console=yes',]
+ boot_params = ' '.join(boot_params)
+ boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params)
+ utils.ReplaceLine('/boot/syslinux/syslinux.cfg',
+ 'APPEND root=',
+ boot_spec)
+
+
+def ForwardSystemdToConsole():
+ utils.LogStep('Installing syslinux bootloader')
+ utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes')
+
+
+def SetupNtpServer():
+ utils.LogStep('Configure NTP')
+ utils.WriteFile('/etc/ntp.conf', 'server metadata.google.internal iburst')
+
+
+def SetupNetwork():
+ utils.LogStep('Setup Networking')
+ utils.SecureDeleteFile('/etc/hostname')
+ utils.WriteFile('/etc/hosts', ETC_HOSTS)
+ utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf',
+ ETC_SYSCTL_D_70_DISABLE_IPV6_CONF)
+ utils.EnableService('dhcpcd.service')
+ utils.EnableService('systemd-networkd.service')
+ utils.EnableService('systemd-networkd-wait-online.service')
+
+
+def SetupSsh():
+ utils.LogStep('Configure SSH')
+ utils.WriteFile('/etc/ssh/sshd_not_to_be_run', 'GOOGLE')
+ utils.SecureDeleteFile('/etc/ssh/ssh_host_key')
+ utils.SecureDeleteFile('/etc/ssh/ssh_host_rsa_key*')
+ utils.SecureDeleteFile('/etc/ssh/ssh_host_dsa_key*')
+ utils.SecureDeleteFile('/etc/ssh/ssh_host_ecdsa_key*')
+ utils.WriteFile('/etc/ssh/ssh_config', ETC_SSH_SSH_CONFIG)
+ utils.Chmod('/etc/ssh/ssh_config', 644)
+ utils.WriteFile('/etc/ssh/sshd_config', ETC_SSH_SSHD_CONFIG)
+ utils.Chmod('/etc/ssh/sshd_config', 644)
+
+
+def SetupFail2ban():
+ utils.LogStep('Configure fail2ban')
+ # http://flexion.org/posts/2012-11-ssh-brute-force-defence.html
+ utils.Pacman(['-S', 'fail2ban'])
+ utils.WriteFile('/etc/fail2ban/jail.local', ETC_FAIL2BAN_JAIL_LOCAL)
+ utils.WriteFile('/etc/fail2ban/jail.d/sshd.conf',
+ ETC_FAIL2BAN_JAIL_D_SSHD_CONF)
+ utils.EnableService('syslog-ng')
+ utils.EnableService('fail2ban.service')
+ utils.EnableService('sshd.service')
+
+
+def ConfigureSecurity():
+ utils.LogStep('Compute Engine Security Recommendations')
+ utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf',
+ ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF)
+ utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf',
+ ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF)
+ utils.LogStep('Lock Root User Account')
+ utils.Run(['usermod', '-L', 'root'])
+ utils.LogStep('PAM Security Settings')
+ utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD)
+
+ utils.LogStep('Disable CAP_SYS_MODULE')
+ utils.WriteFile('/proc/sys/kernel/modules_disabled', '1')
+
+ utils.LogStep('Remove the kernel symbol table')
+ utils.SecureDeleteFile('/boot/System.map')
+
+ utils.LogStep('Sudo Access')
+ utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM)
+ utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm'])
+ utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm'])
+
+
+def ConfigureSerialPortOutput():
+ # https://wiki.archlinux.org/index.php/working_with_the_serial_console
+ # Try this: http://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot
+ utils.LogStep('Configure Serial Port Output')
+
+ utils.Sed('/boot/syslinux/syslinux.cfg', '/DEFAULT/aserial 0 38400')
+ utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1')
+
+
+def InstallGcePackages(packages_dir):
+ try:
+ InstallGoogleCloudSdk()
+ except:
+ pass
+ try:
+ InstallComputeImagePackages(packages_dir)
+ except:
+ pass
+
+
+def InstallComputeImagePackages(packages_dir):
+ utils.LogStep('Install compute-image-packages')
+ utils.Run(["egrep -lRZ 'python' %s | "
+ "xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" %
+ packages_dir],
+ shell=True)
+ utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/')
+ utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'),
+ '/')
+ utils.SecureDeleteFile('/README.md')
+ # TODO: Fix gcimagebundle does not work with Arch yet.
+ #InstallGcimagebundle(packages_dir)
+
+ # Patch Google services to run after the network is actually available.
+ PatchGoogleSystemdService(
+ '/usr/lib/systemd/system/google-startup-scripts.service')
+ PatchGoogleSystemdService(
+ '/usr/lib/systemd/system/google-accounts-manager.service')
+ PatchGoogleSystemdService(
+ '/usr/lib/systemd/system/google-address-manager.service')
+ PatchGoogleSystemdService(
+ '/usr/lib/systemd/system/google.service')
+ utils.EnableService('google-accounts-manager.service')
+ utils.EnableService('google-address-manager.service')
+ utils.EnableService('google.service')
+ utils.EnableService('google-startup-scripts.service')
+ utils.DeleteDirectory(packages_dir)
+
+
+def InstallGcimagebundle(packages_dir):
+ utils.WriteFile(
+ os.path.join(packages_dir, 'gcimagebundle/gcimagebundlelib/arch.py'),
+ GCIMAGEBUNDLE_ARCH_PY)
+ utils.Run(['python2', 'setup.py', 'install'],
+ cwd=os.path.join(packages_dir, 'gcimagebundle'))
+
+
+def PatchGoogleSystemdService(file_path):
+ utils.ReplaceLine(file_path,
+ 'After=network.target', 'After=network-online.target')
+ utils.ReplaceLine(file_path,
+ 'Requires=network.target', 'Requires=network-online.target')
+
+
+def InstallGoogleCloudSdk():
+ # TODO: There's a google-cloud-sdk in AUR which should be used
+ # but it's not optimal for cloud use. The image is too large.
+ utils.LogStep('Install Google Cloud SDK')
+ usr_share_google = '/usr/share/google'
+ archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip')
+ unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk')
+ utils.CreateDirectory(usr_share_google)
+ utils.DownloadFile(
+ 'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip', archive)
+ utils.Run(['unzip', archive, '-d', usr_share_google])
+ utils.AppendFile('/etc/bash.bashrc',
+ 'export CLOUDSDK_PYTHON=/usr/bin/python2')
+ utils.Run([os.path.join(unzip_dir, 'install.sh'),
+ '--usage-reporting', 'false',
+ '--bash-completion', 'true',
+ '--disable-installation-options',
+ '--rc-path', '/etc/bash.bashrc',
+ '--path-update', 'true'],
+ cwd=unzip_dir,
+ env={'CLOUDSDK_PYTHON': '/usr/bin/python2'})
+ utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud')
+ utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil')
+ utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil')
+ utils.SecureDeleteFile(archive)
+
+
+def ConfigMessageOfTheDay():
+ utils.LogStep('Configure Message of the Day')
+ utils.WriteFile('/etc/motd', ETC_MOTD)
+
+
+main()
diff --git a/arch-staging.py b/arch-staging.py
new file mode 100755
index 0000000..344e7b7
--- /dev/null
+++ b/arch-staging.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import logging
+import os
+import sys
+
+import utils
+
+COMPUTE_IMAGE_PACKAGES_GIT_URL = (
+ 'https://github.com/GoogleCloudPlatform/compute-image-packages.git')
+IMAGE_FILE='disk.raw'
+SETUP_PACKAGES_ESSENTIAL = 'grep file'.split()
+SETUP_PACKAGES = 'pacman wget gcc make parted git setconf libaio sudo'.split()
+IMAGE_PACKAGES = ('base tar wget '
+ 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance '
+ 'ntp psmisc openssh udev less bash-completion zip unzip '
+ 'python2 python3').split()
+
+
+def main():
+ args = utils.DecodeArgs(sys.argv[1])
+ utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose'])
+ logging.info('Setup Bootstrapper Environment')
+ utils.SetupArchLocale()
+ InstallPackagesForStagingEnvironment()
+ image_path = os.path.join(os.getcwd(), IMAGE_FILE)
+ CreateImage(image_path, size_gb=int(args['size_gb']))
+ mount_path = utils.CreateTempDirectory(base_dir='/')
+ image_mapping = utils.ImageMapper(image_path, mount_path)
+ try:
+ image_mapping.Map()
+ primary_mapping = image_mapping.GetFirstMapping()
+ image_mapping_path = primary_mapping['path']
+ FormatImage(image_mapping_path)
+ try:
+ image_mapping.Mount()
+ utils.CreateDirectory('/run/shm')
+ utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm'))
+ InstallArchLinux(mount_path)
+ disk_uuid = SetupFileSystem(mount_path, image_mapping_path)
+ ConfigureArchInstall(
+ args, mount_path, primary_mapping['parent'], disk_uuid)
+ utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm'))
+ PurgeDisk(mount_path)
+ finally:
+ image_mapping.Unmount()
+ ShrinkDisk(image_mapping_path)
+ finally:
+ image_mapping.Unmap()
+ utils.Run(['parted', image_path, 'set', '1', 'boot', 'on'])
+ utils.Sync()
+
+
+def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid):
+ relative_builder_path = utils.CopyBuilder(mount_path)
+ utils.LogStep('Download compute-image-packages')
+ packages_dir = utils.CreateTempDirectory(mount_path)
+ utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir])
+ utils.CreateDirectory(os.path.join(mount_path, ''))
+ packages_dir = os.path.relpath(packages_dir, mount_path)
+ params = {
+ 'packages_dir': '/%s' % packages_dir,
+ 'device': parent_path,
+ 'disk_uuid': disk_uuid,
+ 'accounts': args['accounts'],
+ 'debugmode': args['debugmode'],
+ }
+ params.update(args)
+ config_arch_py = os.path.join(
+ '/', relative_builder_path, 'arch-image.py')
+ utils.RunChroot(mount_path,
+ '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)),
+ use_custom_path=False)
+ utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path))
+
+
+def InstallPackagesForStagingEnvironment():
+ utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL)
+ utils.InstallPackages(SETUP_PACKAGES)
+ utils.SetupArchLocale()
+ utils.AurInstall(name='multipath-tools-git')
+ utils.AurInstall(name='zerofree')
+
+
+def CreateImage(image_path, size_gb=10, fs_type='ext4'):
+ utils.LogStep('Create Image')
+ utils.Run(['rm', '-f', image_path])
+ utils.Run(['truncate', image_path, '--size=%sG' % size_gb])
+ utils.Run(['parted', image_path, 'mklabel', 'msdos'])
+ utils.Run(['parted', image_path, 'mkpart', 'primary',
+ fs_type, '1', str(int(size_gb) * 1024)])
+
+
+def FormatImage(image_mapping_path):
+ utils.LogStep('Format Image')
+ utils.Run(['mkfs', image_mapping_path])
+ utils.Sync()
+
+
+def InstallArchLinux(base_dir):
+ utils.LogStep('Install Arch Linux')
+ utils.Pacstrap(base_dir, IMAGE_PACKAGES)
+
+
+def SetupFileSystem(base_dir, image_mapping_path):
+ utils.LogStep('File Systems')
+ _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir],
+ capture_output=True)
+ utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents)
+ _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID',
+ '-o', 'value',
+ image_mapping_path],
+ capture_output=True)
+ disk_uuid = disk_uuid.strip()
+ utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'),
+ 'UUID=%s / ext4 defaults 0 1' % disk_uuid)
+ utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path])
+ return disk_uuid
+
+
+def PurgeDisk(mount_path):
+ paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync']
+ for path in paths:
+ utils.DeleteDirectory(os.path.join(mount_path, path))
+
+
+def ShrinkDisk(image_mapping_path):
+ utils.LogStep('Shrink Disk')
+ utils.Run(['zerofree', image_mapping_path])
+
+
+main()
diff --git a/build-gce-arch.py b/build-gce-arch.py
new file mode 100755
index 0000000..ee487b6
--- /dev/null
+++ b/build-gce-arch.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Arch Linux Image Builder for GCE.
+
+This script creates a clean Arch Linux image that can be used in Google Compute
+Engine.
+
+Usage: ./build-gce-arch.py -> archlinux-***.tar.gz
+ ./build-gce-arch.py --packages docker
+ ./build-gce-arch.py --help -> Detailed help.
+"""
+import argparse
+import os
+import logging
+from datetime import date
+
+import utils
+
+
+DEFAULT_MIRROR = 'http://mirrors.kernel.org/archlinux/$repo/os/$arch'
+#DEFAULT_MIRROR = 'http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch'
+TARGET_ARCH = 'x86_64'
+
+
+def main():
+ args = ParseArgs()
+ utils.SetupLogging(quiet=args.quiet, verbose=args.verbose)
+ workspace_dir = None
+ image_file = None
+ try:
+ workspace_dir = utils.CreateTempDirectory()
+ bootstrap_file = DownloadArchBootstrap(args.bootstrap)
+ utils.Untar(bootstrap_file, workspace_dir)
+ arch_root = PrepareBootstrap(workspace_dir, args.mirror, not args.nopacmankeys)
+ relative_builder_path = utils.CopyBuilder(arch_root)
+ ChrootIntoArchAndBuild(arch_root, relative_builder_path, args)
+ image_name, image_filename, image_description = GetImageNameAndDescription(
+ args.outfile)
+ image_file = SaveImage(arch_root, image_filename)
+ if args.upload and image_file:
+ UploadImage(image_file, args.upload, make_public=args.public)
+ if args.register:
+ AddImageToComputeEngineProject(
+ image_name, args.upload, image_description)
+ finally:
+ if not args.nocleanup and workspace_dir:
+ utils.DeleteDirectory(workspace_dir)
+
+
+def ParseArgs():
+ parser = argparse.ArgumentParser(
+ description='Arch Linux Image Builder for Compute Engine')
+ parser.add_argument('-p', '--packages',
+ dest='packages',
+ nargs='+',
+ help='Additional packages to install via Pacman.')
+ parser.add_argument('--mirror',
+ dest='mirror',
+ default=DEFAULT_MIRROR,
+ help='Mirror to download packages from.')
+ parser.add_argument('--bootstrap',
+ dest='bootstrap',
+ help='Arch Linux Bootstrap tarball. '
+ '(default: Download latest version)')
+ parser.add_argument('-v', '--verbose',
+ dest='verbose',
+ default=False,
+ help='Verbose console output.',
+ action='store_true')
+ parser.add_argument('-q', '--quiet',
+ dest='quiet',
+ default=False,
+ help='Suppress all console output.',
+ action='store_true')
+ parser.add_argument('--upload',
+ dest='upload',
+ default=None,
+ help='Google Cloud Storage path to upload to.')
+ parser.add_argument('--size_gb',
+ dest='size_gb',
+ default=10,
+ help='Volume size of image (in GiB).')
+ parser.add_argument('--accounts',
+ dest='accounts',
+ nargs='+',
+ help='Space delimited list of user accounts to create on '
+ 'the image. Format: username:password')
+ parser.add_argument('--nocleanup',
+ dest='nocleanup',
+ default=False,
+ help='Prevent cleaning up the image build workspace '
+ 'after image has been created.',
+ action='store_true')
+ parser.add_argument('--outfile',
+ dest='outfile',
+ default=None,
+ help='Name of the output image file.')
+ parser.add_argument('--debug',
+ dest='debug',
+ default=False,
+ help='Configure the image for debugging.',
+ action='store_true')
+ parser.add_argument('--public',
+ dest='public',
+ default=False,
+ help='Make image file uploaded to Cloud Storage '
+ 'available for everyone.',
+ action='store_true')
+ parser.add_argument('--register',
+ dest='register',
+ default=False,
+ help='Add the image to Compute Engine project. '
+ '(Upload to Cloud Storage required.)',
+ action='store_true')
+ parser.add_argument('--nopacmankeys',
+ dest='nopacmankeys',
+ default=False,
+ help='Disables signature checking for pacman packages.',
+ action='store_true')
+ return parser.parse_args()
+
+
+def DownloadArchBootstrap(bootstrap_tarball):
+ utils.LogStep('Download Arch Linux Bootstrap')
+ if bootstrap_tarball:
+ url = bootstrap_tarball
+ sha1sum = None
+ else:
+ url, sha1sum = GetLatestBootstrapUrl()
+ logging.debug('Downloading %s', url)
+ local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url))
+ if os.path.isfile(local_bootstrap):
+ logging.debug('Using local file instead.')
+ if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum:
+ return local_bootstrap
+ utils.DownloadFile(url, local_bootstrap)
+ if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum:
+ raise ValueError('Bad checksum')
+ return local_bootstrap
+
+
+def ChrootIntoArchAndBuild(arch_root, relative_builder_path, args):
+ params = {
+ 'quiet': args.quiet,
+ 'verbose': args.verbose,
+ 'packages': args.packages,
+ 'mirror': args.mirror,
+ 'accounts': args.accounts,
+ 'debugmode': args.debug,
+ 'size_gb': args.size_gb
+ }
+ chroot_archenv_script = os.path.join('/', relative_builder_path,
+ 'arch-staging.py')
+ utils.RunChroot(arch_root,
+ '%s "%s"' % (chroot_archenv_script, utils.EncodeArgs(params)))
+ logging.debug('Bootstrap Chroot: sudo %s/bin/arch-chroot %s/',
+ arch_root, arch_root)
+
+
+def SaveImage(arch_root, image_filename):
+ utils.LogStep('Save Arch Linux Image in GCE format')
+ source_image_raw = os.path.join(arch_root, 'disk.raw')
+ image_raw = os.path.join(os.getcwd(), 'disk.raw')
+ image_file = os.path.join(os.getcwd(), image_filename)
+ utils.Run(['cp', '--sparse=always', source_image_raw, image_raw])
+ utils.Run(['tar', '-Szcf', image_file, 'disk.raw'])
+ return image_file
+
+
+def UploadImage(image_path, gs_path, make_public=False):
+ utils.LogStep('Upload Image to Cloud Storage')
+ utils.SecureDeleteFile('~/.gsutil/*.url')
+ utils.Run(['gsutil', 'rm', gs_path])
+ utils.Run(['gsutil', 'cp', image_path, gs_path])
+ if make_public:
+ utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path])
+
+
+def AddImageToComputeEngineProject(image_name, gs_path, description):
+ utils.LogStep('Add image to project')
+ utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q'])
+ utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q',
+ '--source-uri', gs_path,
+ '--description', description])
+
+def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys):
+ utils.LogStep('Setting up Bootstrap Environment')
+ arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0])
+ mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server)
+ utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'),
+ mirrorlist)
+ utils.CreateDirectory(os.path.join(arch_root, 'run/shm'))
+ if use_pacman_keys:
+ utils.RunChroot(arch_root, 'pacman-key --init')
+ utils.RunChroot(arch_root, 'pacman-key --populate archlinux')
+ else:
+ utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never')
+ # Install the most basic utilities for the bootstrapper.
+ utils.RunChroot(arch_root,
+ 'pacman --noconfirm -Sy python3')
+
+ return arch_root
+
+
+def GetLatestBootstrapUrl():
+ base_url = 'http://mirrors.kernel.org/archlinux/iso/latest/'
+ sha1sums = utils.HttpGet(base_url + 'sha1sums.txt')
+ items = sha1sums.splitlines()
+ for item in items:
+ if TARGET_ARCH in item and 'bootstrap' in item:
+ entries = item.split()
+ return base_url + entries[1], entries[0]
+ raise RuntimeError('Cannot find Arch bootstrap url')
+
+
+def GetImageNameAndDescription(outfile_name):
+ today = date.today()
+ isodate = today.strftime("%Y-%m-%d")
+ yyyymmdd = today.strftime("%Y%m%d")
+ image_name = 'arch-v%s' % yyyymmdd
+ if outfile_name:
+ image_filename = outfile_name
+ else:
+ image_filename = '%s.tar.gz' % image_name
+ description = 'Arch Linux x86-64 built on %s' % isodate
+ return image_name, image_filename, description
+
+
+main()
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000..dec05da
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,354 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import glob
+import gzip
+import hashlib
+import json
+import os
+import logging
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.request, urllib.error, urllib.parse
+
+
+APP_NAME = 'archbuilder'
+ETC_LOCALE_GEN = '''
+en_US.UTF-8 UTF-8
+en_US ISO-8859-1
+'''
+
+
+def LogStep(msg):
+ logging.info('- %s', msg)
+
+
+def SetupLogging(quiet=False, verbose=False):
+ if not quiet:
+ root = logging.getLogger()
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ if verbose:
+ stdout_logger.setLevel(logging.DEBUG)
+ else:
+ stdout_logger.setLevel(logging.WARNING)
+ stdout_logger.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+ root.addHandler(stdout_logger)
+ root.setLevel(logging.DEBUG)
+
+
+def Sed(file_path, modification):
+ Run(['sed', '-i', modification, file_path])
+
+
+def Replace(file_path, pattern, replacement):
+ Sed(file_path, 's/%s/%s/g' % (pattern, replacement))
+
+
+def ReplaceLine(file_path, pattern, replacement):
+ Sed(file_path, '/%s/c\%s' % (pattern, replacement))
+
+
+def SudoRun(params, cwd=None, capture_output=False):
+ if os.geteuid() != 0:
+ params = ['sudo'] + params
+ return Run(params, capture_output=capture_output)
+
+
+def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True):
+ try:
+ logging.debug(params)
+ if env:
+ new_env = os.environ.copy()
+ new_env.update(env)
+ env = new_env
+ out_pipe = None
+ if capture_output:
+ out_pipe = subprocess.PIPE
+ proc = subprocess.Popen(params, stdout=out_pipe, cwd=cwd, shell=shell, env=env)
+ if not wait:
+ return 0, '', ''
+ out, err = proc.communicate()
+ if capture_output:
+ logging.debug(out)
+
+ if out:
+ out = out.decode(encoding='UTF-8')
+ if err:
+ err = err.decode(encoding='UTF-8')
+ except KeyboardInterrupt:
+ return 1, '', ''
+ except:
+ logging.error(sys.exc_info()[0])
+ logging.error(sys.exc_info())
+ Run(['/bin/bash'])
+ return 1, '', ''
+
+ return proc.returncode, out, err
+
+
+def DownloadFile(url, file_path):
+ Run(['wget', '-O', file_path, url])
+
+
+def HttpGet(url):
+ return str(urllib.request.urlopen(url).read(), encoding='utf8')
+
+
+def Sha1Sum(file_path):
+ with open(file_path, 'rb') as fp:
+ return hashlib.sha1(fp.read()).hexdigest()
+
+
+def Untar(file_path, dest_dir):
+ Run(['tar', '-C', dest_dir, '-xzf', file_path])
+
+
+def Chmod(fp, mode):
+ Run(['chmod', str(mode), fp])
+
+
+def CreateDirectory(dir_path):
+ dir_path = ToAbsolute(dir_path)
+ if not os.path.isdir(dir_path):
+ os.makedirs(dir_path)
+
+
+def ToAbsolute(path):
+ if not path:
+ return path
+ return os.path.expandvars(os.path.expanduser(path))
+
+
+def DeleteFile(file_pattern):
+ DeleteFileFunc(file_pattern, lambda item: os.remove(item))
+
+
+def SecureDeleteFile(file_pattern):
+ DeleteFileFunc(file_pattern, lambda item: Run(['shred', '--remove', '--zero', item]))
+
+
+def DeleteFileFunc(file_pattern, delete_func):
+ items = glob.glob(ToAbsolute(file_pattern))
+ for item in items:
+ logging.warning('Deleting %s', item)
+ delete_func(item)
+
+
+def DeleteDirectory(dir_path):
+ shutil.rmtree(dir_path)
+
+
+def CreateTempDirectory(base_dir=None):
+ return tempfile.mkdtemp(dir=ToAbsolute(base_dir), prefix='gcearch')
+
+
+def WriteFile(path, content):
+ with open(ToAbsolute(path), 'w') as fp:
+ fp.write(content)
+
+
+def AppendFile(path, content):
+ with open(ToAbsolute(path), 'a') as fp:
+ fp.write(content)
+
+
+def RunChroot(base_dir, command, use_custom_path=True):
+ base_dir = ToAbsolute(base_dir)
+ if use_custom_path:
+ chroot_file = os.path.join(base_dir, 'bin/arch-chroot')
+ else:
+ chroot_file = 'arch-chroot'
+ SudoRun([chroot_file, base_dir, '/bin/bash', '-c', command])
+
+
+def CopyFiles(source_pattern, dest):
+ """Copies a set of files based on glob pattern to a directory.
+ Avoiding shutil.copyfile because of bugs.python.org/issue10016."""
+ items = glob.glob(ToAbsolute(source_pattern))
+ for item in items:
+ Run(['cp', '-Rf', item, dest])
+
+
+def CopyBuilder(base_dir):
+ script_dir = os.path.dirname(os.path.realpath(__file__))
+ temp_dir = CreateTempDirectory(base_dir=base_dir)
+ DeleteDirectory(temp_dir)
+ relative_dir = os.path.relpath(temp_dir, base_dir)
+ shutil.copytree(script_dir, temp_dir, ignore=shutil.ignore_patterns('*.tar.gz', '*.raw'))
+ return relative_dir
+
+
+def EncodeArgs(decoded_args):
+ return base64.standard_b64encode(gzip.compress(bytes(json.dumps(decoded_args), 'utf-8'))).decode('utf-8')
+
+
+def DecodeArgs(encoded_args):
+ return json.loads(gzip.decompress(base64.standard_b64decode(encoded_args)).decode('utf-8'))
+
+
+def DebugBash():
+ Run(['/bin/bash'])
+
+
+def DebugPrintFile(file_path):
+ logging.info('==================================================================================')
+ logging.info('File: %s', file_path)
+ logging.info('==================================================================================')
+ Run(['cat', file_path])
+
+
+def Sync():
+ Run(['sync'])
+
+
+def EnableService(service_name):
+ Run(['systemctl', 'enable', service_name])
+
+
+def Symlink(source_file, dest_file):
+ Run(['ln', '-s', source_file, dest_file])
+
+
+def AurInstall(name=None, pkbuild_url=None):
+ if name:
+ pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower())
+ workspace_dir = CreateTempDirectory()
+ DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD'))
+ Run(['makepkg', '--asroot'], cwd=workspace_dir)
+ tarball = glob.glob(os.path.join(workspace_dir, '*.tar*'))
+ tarball = tarball[0]
+ Pacman(['-U', tarball], cwd=workspace_dir)
+ return tarball
+
+
+def Pacstrap(base_dir, params):
+ Run(['pacstrap', base_dir] + params)
+
+
+def Pacman(params, cwd=None):
+ #, '--debug'
+ Run(['pacman', '--noconfirm'] + params, cwd=cwd)
+
+
+def InstallPackages(package_list):
+ Pacman(['-S'] + package_list)
+
+
+def SetupArchLocale():
+ AppendFile('/etc/locale.gen', ETC_LOCALE_GEN)
+ Run(['locale-gen'])
+ Run(['localectl', 'set-locale', 'LANG="en_US.UTF-8"', 'LC_COLLATE="C"'])
+
+
+class ImageMapper(object):
+ """Interface for kpartx, mount, and umount."""
+ def __init__(self, raw_disk, mount_path):
+ self._raw_disk = raw_disk
+ self._mount_path = mount_path
+ self._device_map = None
+ self._mount_points = None
+
+ def _LoadPartitionsIfNeeded(self):
+ if not self._device_map:
+ self.LoadPartitions()
+
+ def LoadPartitions(self):
+ return_code, out, err = SudoRun(['kpartx', '-l', self._raw_disk], capture_output=True)
+ # Expected Format
+ # loop2p1 : 0 10483712 /dev/loop2 2048
+ # loop2p2 : 0 1 /dev/loop2 2047
+ # loop deleted : /dev/loop2
+
+ self._device_map = {}
+ lines = out.splitlines()
+ for line in lines:
+ parts = str(line).split()
+ if len(parts) == 6:
+ mapping = {
+ 'name': parts[0],
+ 'size_blocks': parts[3],
+ 'parent': parts[4],
+ 'start_block': parts[5],
+ 'path': '/dev/mapper/%s' % str(parts[0])
+ }
+ logging.info('Mapping: %s', mapping)
+ self._device_map[mapping['name']] = mapping
+ if len(self._device_map) == 1:
+ self._mount_points = [self._mount_path]
+ else:
+ self._mount_points = []
+ for name in list(self._device_map.keys()):
+ self._mount_points.append(os.path.join(self._mount_path, name))
+
+ def ForEachDevice(self, func):
+ for name in list(self._device_map.keys()):
+ spec = self._device_map[name]
+ func(spec)
+
+ def ForEachDeviceWithIndex(self, func):
+ i = 0
+ for name in list(self._device_map.keys()):
+ spec = self._device_map[name]
+ func(i, spec)
+ i += 1
+
+ def GetFirstMapping(self):
+ logging.info('DeviceMap: %s', self.GetDeviceMap())
+ return next(iter(self.GetDeviceMap().values()))
+
+ def GetDeviceMap(self):
+ return self._device_map
+
+ def Sync(self):
+ Run(['sync'])
+
+ def Map(self):
+ SudoRun(['kpartx', '-a', '-v', '-s', self._raw_disk])
+ self.LoadPartitions()
+
+ def Unmap(self):
+ self.Sync()
+ time.sleep(2)
+ SudoRun(['kpartx', '-d', '-v', '-s', self._raw_disk])
+ self._device_map = None
+
+ def Mount(self):
+ self._LoadPartitionsIfNeeded()
+ self._ThrowIfBadMountMap()
+ def MountCmd(index, spec):
+ mount_point = self._mount_points[index]
+ CreateDirectory(mount_point)
+ SudoRun(['mount', spec['path'], mount_point])
+ self.ForEachDeviceWithIndex(MountCmd)
+
+ def Unmount(self):
+ self._LoadPartitionsIfNeeded()
+ self._ThrowIfBadMountMap()
+ for path in self._mount_points:
+ SudoRun(['umount', path])
+ self.Sync()
+ time.sleep(2)
+
+ def _ThrowIfBadMountMap(self):
+ if not self._mount_points:
+ raise IOError('Attempted to found {0} without a mount points.'.format(self._raw_disk))
+ if len(self._mount_points) != len(list(self._device_map.keys())):
+ raise IOError('Number of device maps ({0}) does not match mount points ({1}).'.format(
+ len(list(self._device_map.keys())), len(self._mount_points)
+ ))