diff options
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | CONTRIB.md | 64 | ||||
| -rw-r--r-- | LICENSE | 203 | ||||
| -rw-r--r-- | README.md | 60 | ||||
| -rwxr-xr-x | arch-image.py | 521 | ||||
| -rwxr-xr-x | arch-staging.py | 147 | ||||
| -rwxr-xr-x | build-gce-arch.py | 244 | ||||
| -rw-r--r-- | utils.py | 354 |
8 files changed, 1599 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..951b277 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# Files to ignore +*.log +*.tar.gz +*.pyc +*.raw +__pycache__/* diff --git a/CONTRIB.md b/CONTRIB.md new file mode 100644 index 0000000..b368291 --- /dev/null +++ b/CONTRIB.md @@ -0,0 +1,64 @@ +# How to become a contributor and submit your own code + +## Contributor License Agreements + +We'd love to accept your sample apps and patches! Before we can take them, we +have to jump a couple of legal hurdles. + +Please fill out either the individual or corporate Contributor License Agreement +(CLA). + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual CLA] + (https://developers.google.com/open-source/cla/individual). + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA] + (https://developers.google.com/open-source/cla/corporate). + +Follow either of the two links above to access the appropriate CLA and +instructions for how to sign and return it. Once we receive it, we'll be able to +accept your pull requests. + +## Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +1. The repo owner will respond to your issue promptly. +1. If your proposed change is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Fork the desired repo, develop and test your code changes. +1. Ensure that your code adheres to the existing style in the sample to which + you are contributing. Refer to the + [Google Cloud Platform Samples Style Guide] + (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + recommended coding standards for this organization. +1. Ensure that your code has an appropriate set of unit tests which all pass. +1. Submit a pull request. + +## Contributing A New Sample App + +1. Submit an issue to the GoogleCloudPlatform/Template repo describing your + proposed sample app. +1. The Template repo owner will respond to your enhancement issue promptly. + Instructional value is the top priority when evaluating new app proposals for + this collection of repos. +1. If your proposal is accepted, and you haven't already done so, sign a + Contributor License Agreement (see details above). +1. Create your own repo for your app following this naming convention: + * {product}-{app-name}-{language} + * products: appengine, compute, storage, bigquery, prediction, cloudsql + * example: appengine-guestbook-python + * For multi-product apps, concatenate the primary products, like this: + compute-appengine-demo-suite-python. + * For multi-language apps, concatenate the primary languages like this: + appengine-sockets-python-java-go. + +1. Clone the README.md, CONTRIB.md and LICENSE files from the + GoogleCloudPlatform/Template repo. +1. Ensure that your code adheres to the existing style in the sample to which + you are contributing. Refer to the + [Google Cloud Platform Samples Style Guide] + (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the + recommended coding standards for this organization. +1. Ensure that your code has an appropriate set of unit tests which all pass. +1. Submit a request to fork your repo in GoogleCloudPlatform organizationt via + your proposal issue.
\ No newline at end of file @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +
\ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f3f628e --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +## Arch Linux Image Builder for GCE + +Creates an Arch Linux image that can run on Google Compute Engine. + +The image is configured close to the recommendations listed on [Building an image from scratch](https://developers.google.com/compute/docs/images#buildingimage). + +These scripts are written in Python3. + +## Usage + +### Install and Configure Cloud SDK (one time setup) +``` +# Install Cloud SDK (https://developers.google.com/cloud/sdk/) +# For linux: +curl https://sdk.cloud.google.com | bash + +gcloud auth login +gcloud config set project <project> +# Your project ID in Cloud Console, https://console.developers.google.com/ +``` + +### On a Compute Engine VM (recommended) +``` +./build-arch-on-gce.sh --upload gs://${BUCKET}/archlinux.tar.gz + +# You will need a Cloud Storage bucket. +# List buckets owned by your project. +gsutil ls gs:// +# Create a new bucket +gsutil mb gs://${BUCKET} +``` + +### Locally +``` +# Install Required Packages +# Arch Linux +sudo pacman -S python haveged +# Debian +sudo apt-get -y install python3 haveged +# Redhat +sudo yum install -y python3 haveged + +./build-gce-arch.py --verbose +# Upload to Cloud Storage +gsutil cp archlinux-gce.tar.gz gs://${BUCKET}/archlinux.tar.gz + +# Add image to project +gcloud compute images insert archlinux \ + --source-uri gs://${BUCKET}/archlinux.tar.gz \ + --description "Arch Linux for Compute Engine" +``` + + +## Contributing changes + +* See [CONTRIB.md](CONTRIB.md) + + +## Licensing +All files in this repository are under the [Apache License, Version 2.0](LICENSE) unless noted otherwise. diff --git a/arch-image.py b/arch-image.py new file mode 100755 index 0000000..04bf6e4 --- /dev/null +++ b/arch-image.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys + +import utils + + +ETC_MOTD = '''Arch Linux for Compute Engine +''' + +ETC_HOSTS = '''127.0.0.1 localhost +169.254.169.254 metadata.google.internal metadata +''' + +ETC_SSH_SSH_CONFIG = ''' +Host * +Protocol 2 +ForwardAgent no +ForwardX11 no +HostbasedAuthentication no +StrictHostKeyChecking no +Ciphers aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc +Tunnel no + +# Google Compute Engine times out connections after 10 minutes of inactivity. +# Keep alive ssh connections by sending a packet every 7 minutes. +ServerAliveInterval 420 +''' + +ETC_SSH_SSHD_CONFIG = ''' +# Disable PasswordAuthentication as ssh keys are more secure. +PasswordAuthentication no + +# Disable root login, using sudo provides better auditing. +PermitRootLogin no + +PermitTunnel no +AllowTcpForwarding yes +X11Forwarding no + +# Compute times out connections after 10 minutes of inactivity. Keep alive +# ssh connections by sending a packet every 7 minutes. +ClientAliveInterval 420 + +# Restrict sshd to just IPv4 for now as sshd gets confused for things +# like X11 forwarding. + +Port 22 +Protocol 2 + +UsePrivilegeSeparation yes + +# Lifetime and size of ephemeral version 1 server key +KeyRegenerationInterval 3600 +ServerKeyBits 768 + +SyslogFacility AUTH +LogLevel INFO + +LoginGraceTime 120 +StrictModes yes + +RSAAuthentication yes +PubkeyAuthentication yes + +IgnoreRhosts yes +RhostsRSAAuthentication no +HostbasedAuthentication no + +PermitEmptyPasswords no +ChallengeResponseAuthentication no + +PasswordAuthentication no +PrintMotd no +PrintLastLog yes + +TCPKeepAlive yes + +Subsystem sftp /usr/lib/openssh/sftp-server + +UsePAM yes +UseDNS no +''' + +ETC_SYSCTL_D_70_DISABLE_IPV6_CONF = ''' +net.ipv6.conf.all.disable_ipv6 = 1 +''' + +ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF = ''' +# enables syn flood protection +net.ipv4.tcp_syncookies = 1 + +# ignores source-routed packets +net.ipv4.conf.all.accept_source_route = 0 + +# ignores source-routed packets +net.ipv4.conf.default.accept_source_route = 0 + +# ignores ICMP redirects +net.ipv4.conf.all.accept_redirects = 0 + +# ignores ICMP redirects +net.ipv4.conf.default.accept_redirects = 0 + +# ignores ICMP redirects from non-GW hosts +net.ipv4.conf.all.secure_redirects = 1 + +# ignores ICMP redirects from non-GW hosts +net.ipv4.conf.default.secure_redirects = 1 + +# don't allow traffic between networks or act as a router +net.ipv4.ip_forward = 0 + +# don't allow traffic between networks or act as a router +net.ipv4.conf.all.send_redirects = 0 + +# don't allow traffic between networks or act as a router +net.ipv4.conf.default.send_redirects = 0 + +# reverse path filtering - IP spoofing protection +net.ipv4.conf.all.rp_filter = 1 + +# reverse path filtering - IP spoofing protection +net.ipv4.conf.default.rp_filter = 1 + +# reverse path filtering - IP spoofing protection +net.ipv4.conf.default.rp_filter = 1 + +# ignores ICMP broadcasts to avoid participating in Smurf attacks +net.ipv4.icmp_echo_ignore_broadcasts = 1 + +# ignores bad ICMP errors +net.ipv4.icmp_ignore_bogus_error_responses = 1 + +# logs spoofed, source-routed, and redirect packets +net.ipv4.conf.all.log_martians = 1 + +# log spoofed, source-routed, and redirect packets +net.ipv4.conf.default.log_martians = 1 + +# implements RFC 1337 fix +net.ipv4.tcp_rfc1337 = 1 + +# randomizes addresses of mmap base, heap, stack and VDSO page +kernel.randomize_va_space = 2 +''' + +ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF = ''' +# provides protection from ToCToU races +fs.protected_hardlinks=1 + +# provides protection from ToCToU races +fs.protected_symlinks=1 + +# makes locating kernel addresses more difficult +kernel.kptr_restrict=1 + +# set ptrace protections +kernel.yama.ptrace_scope=1 + +# set perf only available to root +kernel.perf_event_paranoid=2 +''' + +ETC_PAM_D_PASSWD = ''' +#%PAM-1.0 +password required pam_cracklib.so difok=2 minlen=8 dcredit=2 ocredit=2 retry=3 +password required pam_unix.so sha512 shadow nullok +password required pam_tally.so even_deny_root_account deny=3 lock_time=5 unlock_time=3600 +''' + +ETC_SUDOERS_D_ADD_GROUP_ADM = ''' +%adm ALL=(ALL) ALL +''' + +ETC_FAIL2BAN_JAIL_LOCAL = ''' +[DEFAULT] +backend = systemd +loglevel = WARNING +''' + +ETC_FAIL2BAN_JAIL_D_SSHD_CONF = ''' +# fail2ban SSH +# block ssh after 3 unsuccessful login attempts for 10 minutes +[sshd] +enabled = true +action = iptables[chain=INPUT, protocol=tcp, port=22, name=sshd] +maxRetry = 3 +findtime = 600 +bantime = 600 +port = 22 +''' + +GCIMAGEBUNDLE_ARCH_PY = ''' +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Arch Linux specific platform info.""" + + +import os + +from gcimagebundlelib import linux + + +class Arch(linux.LinuxPlatform): + """Arch Linux specific information.""" + + @staticmethod + def IsThisPlatform(root='/'): + return os.path.isfile('/etc/arch-release') + + def __init__(self): + super(Arch, self).__init__() +''' + + +def main(): + args = utils.DecodeArgs(sys.argv[1]) + utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) + logging.info('Setup Bootstrapper Environment') + SetupLocale() + ConfigureTimeZone() + ConfigureKernel() + InstallBootloader(args['device'], args['disk_uuid'], args['debugmode']) + ForwardSystemdToConsole() + SetupNtpServer() + SetupNetwork() + SetupSsh() + #SetupFail2ban() + SetupAccounts(args) + InstallGcePackages(args['packages_dir']) + ConfigMessageOfTheDay() + ConfigureSecurity() + ConfigureSerialPortOutput() + OptimizePackages() + + +def SetupAccounts(args): + accounts = args['accounts'] + if accounts: + utils.LogStep('Add Accounts') + for account in accounts: + username, password = account.split(':') + logging.info(' - %s', username) + utils.Run(['useradd', username, '-m', '-s', '/bin/bash', + '-G', 'adm,video']) + utils.Run('echo %s:%s | chpasswd' % (username, password), shell=True) + + +def OptimizePackages(): + utils.LogStep('Cleanup Cached Package Data') + utils.Pacman(['-Syu']) + utils.Pacman(['-Sc']) + utils.Run(['pacman-optimize']) + + +def SetupLocale(): + utils.LogStep('Set Locale to US English (UTF-8)') + utils.SetupArchLocale() + + +def ConfigureTimeZone(): + utils.LogStep('Set Timezone to UTC') + utils.Run(['ln', '-sf', '/usr/share/zoneinfo/UTC', '/etc/localtime']) + + +def ConfigureKernel(): + utils.LogStep('Configure Kernel') + utils.Replace('/etc/mkinitcpio.conf', + 'MODULES=""', + 'MODULES="virtio virtio_blk virtio_pci virtio_scsi virtio_net"') + utils.Replace('/etc/mkinitcpio.conf', 'autodetect ', '') + utils.Run(['mkinitcpio', + '-g', '/boot/initramfs-linux.img', + '-k', '/boot/vmlinuz-linux', + '-c', '/etc/mkinitcpio.conf']) + + +def InstallBootloader(device, uuid, debugmode): + utils.LogStep('Install Syslinux bootloader') + ''' + utils.Run(['syslinux-install_update', '-i', '-a', '-m']) + ''' + utils.Run(['blkid', '-s', 'PTTYPE', '-o', 'value', device]) + utils.CreateDirectory('/boot/syslinux') + utils.CopyFiles('/usr/lib/syslinux/bios/*.c32', '/boot/syslinux/') + utils.Run(['extlinux', '--install', '/boot/syslinux']) + utils.Replace('/boot/syslinux/syslinux.cfg', 'sda3', 'sda1') + utils.Run(['fdisk', '-l', device]) + utils.Run(['dd', 'bs=440', 'count=1', 'conv=notrunc', + 'if=/usr/lib/syslinux/bios/mbr.bin', 'of=%s' % device]) + + boot_params = [ + 'console=ttyS0,38400', + 'CONFIG_KVM_GUEST=y', + 'CONFIG_KVM_CLOCK=y', + 'CONFIG_VIRTIO_PCI=y', + 'CONFIG_SCSI_VIRTIO=y', + 'CONFIG_VIRTIO_NET=y', + 'CONFIG_STRICT_DEVMEM=y', + 'CONFIG_DEVKMEM=n', + 'CONFIG_DEFAULT_MMAP_MIN_ADDR=65536', + 'CONFIG_DEBUG_RODATA=y', + 'CONFIG_DEBUG_SET_MODULE_RONX=y', + 'CONFIG_CC_STACKPROTECTOR=y', + 'CONFIG_COMPAT_VDSO=n', + 'CONFIG_COMPAT_BRK=n', + 'CONFIG_X86_PAE=y', + 'CONFIG_SYN_COOKIES=y', + 'CONFIG_SECURITY_YAMA=y', + 'CONFIG_SECURITY_YAMA_STACKED=y', + ] + if debugmode: + boot_params += [ + 'systemd.log_level=debug', + 'systemd.log_target=console', + 'systemd.journald.forward_to_syslog=yes', + 'systemd.journald.forward_to_kmsg=yes', + 'systemd.journald.forward_to_console=yes',] + boot_params = ' '.join(boot_params) + boot_spec = ' APPEND root=UUID=%s rw append %s' % (uuid, boot_params) + utils.ReplaceLine('/boot/syslinux/syslinux.cfg', + 'APPEND root=', + boot_spec) + + +def ForwardSystemdToConsole(): + utils.LogStep('Installing syslinux bootloader') + utils.AppendFile('/etc/systemd/journald.conf', 'ForwardToConsole=yes') + + +def SetupNtpServer(): + utils.LogStep('Configure NTP') + utils.WriteFile('/etc/ntp.conf', 'server metadata.google.internal iburst') + + +def SetupNetwork(): + utils.LogStep('Setup Networking') + utils.SecureDeleteFile('/etc/hostname') + utils.WriteFile('/etc/hosts', ETC_HOSTS) + utils.WriteFile('/etc/sysctl.d/70-disable-ipv6.conf', + ETC_SYSCTL_D_70_DISABLE_IPV6_CONF) + utils.EnableService('dhcpcd.service') + utils.EnableService('systemd-networkd.service') + utils.EnableService('systemd-networkd-wait-online.service') + + +def SetupSsh(): + utils.LogStep('Configure SSH') + utils.WriteFile('/etc/ssh/sshd_not_to_be_run', 'GOOGLE') + utils.SecureDeleteFile('/etc/ssh/ssh_host_key') + utils.SecureDeleteFile('/etc/ssh/ssh_host_rsa_key*') + utils.SecureDeleteFile('/etc/ssh/ssh_host_dsa_key*') + utils.SecureDeleteFile('/etc/ssh/ssh_host_ecdsa_key*') + utils.WriteFile('/etc/ssh/ssh_config', ETC_SSH_SSH_CONFIG) + utils.Chmod('/etc/ssh/ssh_config', 644) + utils.WriteFile('/etc/ssh/sshd_config', ETC_SSH_SSHD_CONFIG) + utils.Chmod('/etc/ssh/sshd_config', 644) + + +def SetupFail2ban(): + utils.LogStep('Configure fail2ban') + # http://flexion.org/posts/2012-11-ssh-brute-force-defence.html + utils.Pacman(['-S', 'fail2ban']) + utils.WriteFile('/etc/fail2ban/jail.local', ETC_FAIL2BAN_JAIL_LOCAL) + utils.WriteFile('/etc/fail2ban/jail.d/sshd.conf', + ETC_FAIL2BAN_JAIL_D_SSHD_CONF) + utils.EnableService('syslog-ng') + utils.EnableService('fail2ban.service') + utils.EnableService('sshd.service') + + +def ConfigureSecurity(): + utils.LogStep('Compute Engine Security Recommendations') + utils.WriteFile('/etc/sysctl.d/70-gce-security-strongly-recommended.conf', + ETC_SYSCTL_D_70_GCE_SECURITY_STRONGLY_RECOMMENDED_CONF) + utils.WriteFile('/etc/sysctl.d/70-gce-security-recommended.conf', + ETC_SYSCTL_D_70_GCE_SECURITY_RECOMMENDED_CONF) + utils.LogStep('Lock Root User Account') + utils.Run(['usermod', '-L', 'root']) + utils.LogStep('PAM Security Settings') + utils.WriteFile('/etc/pam.d/passwd', ETC_PAM_D_PASSWD) + + utils.LogStep('Disable CAP_SYS_MODULE') + utils.WriteFile('/proc/sys/kernel/modules_disabled', '1') + + utils.LogStep('Remove the kernel symbol table') + utils.SecureDeleteFile('/boot/System.map') + + utils.LogStep('Sudo Access') + utils.WriteFile('/etc/sudoers.d/add-group-adm', ETC_SUDOERS_D_ADD_GROUP_ADM) + utils.Run(['chown', 'root:root', '/etc/sudoers.d/add-group-adm']) + utils.Run(['chmod', '0440', '/etc/sudoers.d/add-group-adm']) + + +def ConfigureSerialPortOutput(): + # https://wiki.archlinux.org/index.php/working_with_the_serial_console + # Try this: http://wiki.alpinelinux.org/wiki/Enable_Serial_Console_on_Boot + utils.LogStep('Configure Serial Port Output') + + utils.Sed('/boot/syslinux/syslinux.cfg', '/DEFAULT/aserial 0 38400') + utils.ReplaceLine('/boot/syslinux/syslinux.cfg', 'TIMEOUT', 'TIMEOUT 1') + + +def InstallGcePackages(packages_dir): + try: + InstallGoogleCloudSdk() + except: + pass + try: + InstallComputeImagePackages(packages_dir) + except: + pass + + +def InstallComputeImagePackages(packages_dir): + utils.LogStep('Install compute-image-packages') + utils.Run(["egrep -lRZ 'python' %s | " + "xargs -0 -l sed -i -e '/#!.*python/c\#!/usr/bin/env python2'" % + packages_dir], + shell=True) + utils.CopyFiles(os.path.join(packages_dir, 'google-daemon', '*'), '/') + utils.CopyFiles(os.path.join(packages_dir, 'google-startup-scripts', '*'), + '/') + utils.SecureDeleteFile('/README.md') + # TODO: Fix gcimagebundle does not work with Arch yet. + #InstallGcimagebundle(packages_dir) + + # Patch Google services to run after the network is actually available. + PatchGoogleSystemdService( + '/usr/lib/systemd/system/google-startup-scripts.service') + PatchGoogleSystemdService( + '/usr/lib/systemd/system/google-accounts-manager.service') + PatchGoogleSystemdService( + '/usr/lib/systemd/system/google-address-manager.service') + PatchGoogleSystemdService( + '/usr/lib/systemd/system/google.service') + utils.EnableService('google-accounts-manager.service') + utils.EnableService('google-address-manager.service') + utils.EnableService('google.service') + utils.EnableService('google-startup-scripts.service') + utils.DeleteDirectory(packages_dir) + + +def InstallGcimagebundle(packages_dir): + utils.WriteFile( + os.path.join(packages_dir, 'gcimagebundle/gcimagebundlelib/arch.py'), + GCIMAGEBUNDLE_ARCH_PY) + utils.Run(['python2', 'setup.py', 'install'], + cwd=os.path.join(packages_dir, 'gcimagebundle')) + + +def PatchGoogleSystemdService(file_path): + utils.ReplaceLine(file_path, + 'After=network.target', 'After=network-online.target') + utils.ReplaceLine(file_path, + 'Requires=network.target', 'Requires=network-online.target') + + +def InstallGoogleCloudSdk(): + # TODO: There's a google-cloud-sdk in AUR which should be used + # but it's not optimal for cloud use. The image is too large. + utils.LogStep('Install Google Cloud SDK') + usr_share_google = '/usr/share/google' + archive = os.path.join(usr_share_google, 'google-cloud-sdk.zip') + unzip_dir = os.path.join(usr_share_google, 'google-cloud-sdk') + utils.CreateDirectory(usr_share_google) + utils.DownloadFile( + 'https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip', archive) + utils.Run(['unzip', archive, '-d', usr_share_google]) + utils.AppendFile('/etc/bash.bashrc', + 'export CLOUDSDK_PYTHON=/usr/bin/python2') + utils.Run([os.path.join(unzip_dir, 'install.sh'), + '--usage-reporting', 'false', + '--bash-completion', 'true', + '--disable-installation-options', + '--rc-path', '/etc/bash.bashrc', + '--path-update', 'true'], + cwd=unzip_dir, + env={'CLOUDSDK_PYTHON': '/usr/bin/python2'}) + utils.Symlink(os.path.join(unzip_dir, 'bin/gcloud'), '/usr/bin/gcloud') + utils.Symlink(os.path.join(unzip_dir, 'bin/gcutil'), '/usr/bin/gcutil') + utils.Symlink(os.path.join(unzip_dir, 'bin/gsutil'), '/usr/bin/gsutil') + utils.SecureDeleteFile(archive) + + +def ConfigMessageOfTheDay(): + utils.LogStep('Configure Message of the Day') + utils.WriteFile('/etc/motd', ETC_MOTD) + + +main() diff --git a/arch-staging.py b/arch-staging.py new file mode 100755 index 0000000..344e7b7 --- /dev/null +++ b/arch-staging.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import sys + +import utils + +COMPUTE_IMAGE_PACKAGES_GIT_URL = ( + 'https://github.com/GoogleCloudPlatform/compute-image-packages.git') +IMAGE_FILE='disk.raw' +SETUP_PACKAGES_ESSENTIAL = 'grep file'.split() +SETUP_PACKAGES = 'pacman wget gcc make parted git setconf libaio sudo'.split() +IMAGE_PACKAGES = ('base tar wget ' + 'curl sudo mkinitcpio syslinux dhcp ethtool irqbalance ' + 'ntp psmisc openssh udev less bash-completion zip unzip ' + 'python2 python3').split() + + +def main(): + args = utils.DecodeArgs(sys.argv[1]) + utils.SetupLogging(quiet=args['quiet'], verbose=args['verbose']) + logging.info('Setup Bootstrapper Environment') + utils.SetupArchLocale() + InstallPackagesForStagingEnvironment() + image_path = os.path.join(os.getcwd(), IMAGE_FILE) + CreateImage(image_path, size_gb=int(args['size_gb'])) + mount_path = utils.CreateTempDirectory(base_dir='/') + image_mapping = utils.ImageMapper(image_path, mount_path) + try: + image_mapping.Map() + primary_mapping = image_mapping.GetFirstMapping() + image_mapping_path = primary_mapping['path'] + FormatImage(image_mapping_path) + try: + image_mapping.Mount() + utils.CreateDirectory('/run/shm') + utils.CreateDirectory(os.path.join(mount_path, 'run', 'shm')) + InstallArchLinux(mount_path) + disk_uuid = SetupFileSystem(mount_path, image_mapping_path) + ConfigureArchInstall( + args, mount_path, primary_mapping['parent'], disk_uuid) + utils.DeleteDirectory(os.path.join(mount_path, 'run', 'shm')) + PurgeDisk(mount_path) + finally: + image_mapping.Unmount() + ShrinkDisk(image_mapping_path) + finally: + image_mapping.Unmap() + utils.Run(['parted', image_path, 'set', '1', 'boot', 'on']) + utils.Sync() + + +def ConfigureArchInstall(args, mount_path, parent_path, disk_uuid): + relative_builder_path = utils.CopyBuilder(mount_path) + utils.LogStep('Download compute-image-packages') + packages_dir = utils.CreateTempDirectory(mount_path) + utils.Run(['git', 'clone', COMPUTE_IMAGE_PACKAGES_GIT_URL, packages_dir]) + utils.CreateDirectory(os.path.join(mount_path, '')) + packages_dir = os.path.relpath(packages_dir, mount_path) + params = { + 'packages_dir': '/%s' % packages_dir, + 'device': parent_path, + 'disk_uuid': disk_uuid, + 'accounts': args['accounts'], + 'debugmode': args['debugmode'], + } + params.update(args) + config_arch_py = os.path.join( + '/', relative_builder_path, 'arch-image.py') + utils.RunChroot(mount_path, + '%s "%s"' % (config_arch_py, utils.EncodeArgs(params)), + use_custom_path=False) + utils.DeleteDirectory(os.path.join(mount_path, relative_builder_path)) + + +def InstallPackagesForStagingEnvironment(): + utils.InstallPackages(SETUP_PACKAGES_ESSENTIAL) + utils.InstallPackages(SETUP_PACKAGES) + utils.SetupArchLocale() + utils.AurInstall(name='multipath-tools-git') + utils.AurInstall(name='zerofree') + + +def CreateImage(image_path, size_gb=10, fs_type='ext4'): + utils.LogStep('Create Image') + utils.Run(['rm', '-f', image_path]) + utils.Run(['truncate', image_path, '--size=%sG' % size_gb]) + utils.Run(['parted', image_path, 'mklabel', 'msdos']) + utils.Run(['parted', image_path, 'mkpart', 'primary', + fs_type, '1', str(int(size_gb) * 1024)]) + + +def FormatImage(image_mapping_path): + utils.LogStep('Format Image') + utils.Run(['mkfs', image_mapping_path]) + utils.Sync() + + +def InstallArchLinux(base_dir): + utils.LogStep('Install Arch Linux') + utils.Pacstrap(base_dir, IMAGE_PACKAGES) + + +def SetupFileSystem(base_dir, image_mapping_path): + utils.LogStep('File Systems') + _, fstab_contents, _ = utils.Run(['genfstab', '-p', base_dir], + capture_output=True) + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), fstab_contents) + _, disk_uuid, _ = utils.Run(['blkid', '-s', 'UUID', + '-o', 'value', + image_mapping_path], + capture_output=True) + disk_uuid = disk_uuid.strip() + utils.WriteFile(os.path.join(base_dir, 'etc', 'fstab'), + 'UUID=%s / ext4 defaults 0 1' % disk_uuid) + utils.Run(['tune2fs', '-i', '1', '-U', disk_uuid, image_mapping_path]) + return disk_uuid + + +def PurgeDisk(mount_path): + paths = ['/var/cache', '/var/log', '/var/lib/pacman/sync'] + for path in paths: + utils.DeleteDirectory(os.path.join(mount_path, path)) + + +def ShrinkDisk(image_mapping_path): + utils.LogStep('Shrink Disk') + utils.Run(['zerofree', image_mapping_path]) + + +main() diff --git a/build-gce-arch.py b/build-gce-arch.py new file mode 100755 index 0000000..ee487b6 --- /dev/null +++ b/build-gce-arch.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Arch Linux Image Builder for GCE. + +This script creates a clean Arch Linux image that can be used in Google Compute +Engine. + +Usage: ./build-gce-arch.py -> archlinux-***.tar.gz + ./build-gce-arch.py --packages docker + ./build-gce-arch.py --help -> Detailed help. +""" +import argparse +import os +import logging +from datetime import date + +import utils + + +DEFAULT_MIRROR = 'http://mirrors.kernel.org/archlinux/$repo/os/$arch' +#DEFAULT_MIRROR = 'http://mirror.us.leaseweb.net/archlinux/$repo/os/$arch' +TARGET_ARCH = 'x86_64' + + +def main(): + args = ParseArgs() + utils.SetupLogging(quiet=args.quiet, verbose=args.verbose) + workspace_dir = None + image_file = None + try: + workspace_dir = utils.CreateTempDirectory() + bootstrap_file = DownloadArchBootstrap(args.bootstrap) + utils.Untar(bootstrap_file, workspace_dir) + arch_root = PrepareBootstrap(workspace_dir, args.mirror, not args.nopacmankeys) + relative_builder_path = utils.CopyBuilder(arch_root) + ChrootIntoArchAndBuild(arch_root, relative_builder_path, args) + image_name, image_filename, image_description = GetImageNameAndDescription( + args.outfile) + image_file = SaveImage(arch_root, image_filename) + if args.upload and image_file: + UploadImage(image_file, args.upload, make_public=args.public) + if args.register: + AddImageToComputeEngineProject( + image_name, args.upload, image_description) + finally: + if not args.nocleanup and workspace_dir: + utils.DeleteDirectory(workspace_dir) + + +def ParseArgs(): + parser = argparse.ArgumentParser( + description='Arch Linux Image Builder for Compute Engine') + parser.add_argument('-p', '--packages', + dest='packages', + nargs='+', + help='Additional packages to install via Pacman.') + parser.add_argument('--mirror', + dest='mirror', + default=DEFAULT_MIRROR, + help='Mirror to download packages from.') + parser.add_argument('--bootstrap', + dest='bootstrap', + help='Arch Linux Bootstrap tarball. ' + '(default: Download latest version)') + parser.add_argument('-v', '--verbose', + dest='verbose', + default=False, + help='Verbose console output.', + action='store_true') + parser.add_argument('-q', '--quiet', + dest='quiet', + default=False, + help='Suppress all console output.', + action='store_true') + parser.add_argument('--upload', + dest='upload', + default=None, + help='Google Cloud Storage path to upload to.') + parser.add_argument('--size_gb', + dest='size_gb', + default=10, + help='Volume size of image (in GiB).') + parser.add_argument('--accounts', + dest='accounts', + nargs='+', + help='Space delimited list of user accounts to create on ' + 'the image. Format: username:password') + parser.add_argument('--nocleanup', + dest='nocleanup', + default=False, + help='Prevent cleaning up the image build workspace ' + 'after image has been created.', + action='store_true') + parser.add_argument('--outfile', + dest='outfile', + default=None, + help='Name of the output image file.') + parser.add_argument('--debug', + dest='debug', + default=False, + help='Configure the image for debugging.', + action='store_true') + parser.add_argument('--public', + dest='public', + default=False, + help='Make image file uploaded to Cloud Storage ' + 'available for everyone.', + action='store_true') + parser.add_argument('--register', + dest='register', + default=False, + help='Add the image to Compute Engine project. ' + '(Upload to Cloud Storage required.)', + action='store_true') + parser.add_argument('--nopacmankeys', + dest='nopacmankeys', + default=False, + help='Disables signature checking for pacman packages.', + action='store_true') + return parser.parse_args() + + +def DownloadArchBootstrap(bootstrap_tarball): + utils.LogStep('Download Arch Linux Bootstrap') + if bootstrap_tarball: + url = bootstrap_tarball + sha1sum = None + else: + url, sha1sum = GetLatestBootstrapUrl() + logging.debug('Downloading %s', url) + local_bootstrap = os.path.join(os.getcwd(), os.path.basename(url)) + if os.path.isfile(local_bootstrap): + logging.debug('Using local file instead.') + if sha1sum and utils.Sha1Sum(local_bootstrap) == sha1sum: + return local_bootstrap + utils.DownloadFile(url, local_bootstrap) + if not sha1sum or utils.Sha1Sum(local_bootstrap) != sha1sum: + raise ValueError('Bad checksum') + return local_bootstrap + + +def ChrootIntoArchAndBuild(arch_root, relative_builder_path, args): + params = { + 'quiet': args.quiet, + 'verbose': args.verbose, + 'packages': args.packages, + 'mirror': args.mirror, + 'accounts': args.accounts, + 'debugmode': args.debug, + 'size_gb': args.size_gb + } + chroot_archenv_script = os.path.join('/', relative_builder_path, + 'arch-staging.py') + utils.RunChroot(arch_root, + '%s "%s"' % (chroot_archenv_script, utils.EncodeArgs(params))) + logging.debug('Bootstrap Chroot: sudo %s/bin/arch-chroot %s/', + arch_root, arch_root) + + +def SaveImage(arch_root, image_filename): + utils.LogStep('Save Arch Linux Image in GCE format') + source_image_raw = os.path.join(arch_root, 'disk.raw') + image_raw = os.path.join(os.getcwd(), 'disk.raw') + image_file = os.path.join(os.getcwd(), image_filename) + utils.Run(['cp', '--sparse=always', source_image_raw, image_raw]) + utils.Run(['tar', '-Szcf', image_file, 'disk.raw']) + return image_file + + +def UploadImage(image_path, gs_path, make_public=False): + utils.LogStep('Upload Image to Cloud Storage') + utils.SecureDeleteFile('~/.gsutil/*.url') + utils.Run(['gsutil', 'rm', gs_path]) + utils.Run(['gsutil', 'cp', image_path, gs_path]) + if make_public: + utils.Run(['gsutil', 'acl', 'set', 'public-read', gs_path]) + + +def AddImageToComputeEngineProject(image_name, gs_path, description): + utils.LogStep('Add image to project') + utils.Run(['gcloud', 'compute', 'images', 'delete', image_name, '-q']) + utils.Run(['gcloud', 'compute', 'images', 'create', image_name, '-q', + '--source-uri', gs_path, + '--description', description]) + +def PrepareBootstrap(workspace_dir, mirror_server, use_pacman_keys): + utils.LogStep('Setting up Bootstrap Environment') + arch_root = os.path.join(workspace_dir, os.listdir(workspace_dir)[0]) + mirrorlist = 'Server = {MIRROR_SERVER}'.format(MIRROR_SERVER=mirror_server) + utils.AppendFile(os.path.join(arch_root, 'etc/pacman.d/mirrorlist'), + mirrorlist) + utils.CreateDirectory(os.path.join(arch_root, 'run/shm')) + if use_pacman_keys: + utils.RunChroot(arch_root, 'pacman-key --init') + utils.RunChroot(arch_root, 'pacman-key --populate archlinux') + else: + utils.ReplaceLine(os.path.join(arch_root, 'etc/pacman.conf'), 'SigLevel', 'SigLevel = Never') + # Install the most basic utilities for the bootstrapper. + utils.RunChroot(arch_root, + 'pacman --noconfirm -Sy python3') + + return arch_root + + +def GetLatestBootstrapUrl(): + base_url = 'http://mirrors.kernel.org/archlinux/iso/latest/' + sha1sums = utils.HttpGet(base_url + 'sha1sums.txt') + items = sha1sums.splitlines() + for item in items: + if TARGET_ARCH in item and 'bootstrap' in item: + entries = item.split() + return base_url + entries[1], entries[0] + raise RuntimeError('Cannot find Arch bootstrap url') + + +def GetImageNameAndDescription(outfile_name): + today = date.today() + isodate = today.strftime("%Y-%m-%d") + yyyymmdd = today.strftime("%Y%m%d") + image_name = 'arch-v%s' % yyyymmdd + if outfile_name: + image_filename = outfile_name + else: + image_filename = '%s.tar.gz' % image_name + description = 'Arch Linux x86-64 built on %s' % isodate + return image_name, image_filename, description + + +main() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..dec05da --- /dev/null +++ b/utils.py @@ -0,0 +1,354 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import glob +import gzip +import hashlib +import json +import os +import logging +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request, urllib.error, urllib.parse + + +APP_NAME = 'archbuilder' +ETC_LOCALE_GEN = ''' +en_US.UTF-8 UTF-8 +en_US ISO-8859-1 +''' + + +def LogStep(msg): + logging.info('- %s', msg) + + +def SetupLogging(quiet=False, verbose=False): + if not quiet: + root = logging.getLogger() + stdout_logger = logging.StreamHandler(sys.stdout) + if verbose: + stdout_logger.setLevel(logging.DEBUG) + else: + stdout_logger.setLevel(logging.WARNING) + stdout_logger.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + root.addHandler(stdout_logger) + root.setLevel(logging.DEBUG) + + +def Sed(file_path, modification): + Run(['sed', '-i', modification, file_path]) + + +def Replace(file_path, pattern, replacement): + Sed(file_path, 's/%s/%s/g' % (pattern, replacement)) + + +def ReplaceLine(file_path, pattern, replacement): + Sed(file_path, '/%s/c\%s' % (pattern, replacement)) + + +def SudoRun(params, cwd=None, capture_output=False): + if os.geteuid() != 0: + params = ['sudo'] + params + return Run(params, capture_output=capture_output) + + +def Run(params, cwd=None, capture_output=False, shell=False, env=None, wait=True): + try: + logging.debug(params) + if env: + new_env = os.environ.copy() + new_env.update(env) + env = new_env + out_pipe = None + if capture_output: + out_pipe = subprocess.PIPE + proc = subprocess.Popen(params, stdout=out_pipe, cwd=cwd, shell=shell, env=env) + if not wait: + return 0, '', '' + out, err = proc.communicate() + if capture_output: + logging.debug(out) + + if out: + out = out.decode(encoding='UTF-8') + if err: + err = err.decode(encoding='UTF-8') + except KeyboardInterrupt: + return 1, '', '' + except: + logging.error(sys.exc_info()[0]) + logging.error(sys.exc_info()) + Run(['/bin/bash']) + return 1, '', '' + + return proc.returncode, out, err + + +def DownloadFile(url, file_path): + Run(['wget', '-O', file_path, url]) + + +def HttpGet(url): + return str(urllib.request.urlopen(url).read(), encoding='utf8') + + +def Sha1Sum(file_path): + with open(file_path, 'rb') as fp: + return hashlib.sha1(fp.read()).hexdigest() + + +def Untar(file_path, dest_dir): + Run(['tar', '-C', dest_dir, '-xzf', file_path]) + + +def Chmod(fp, mode): + Run(['chmod', str(mode), fp]) + + +def CreateDirectory(dir_path): + dir_path = ToAbsolute(dir_path) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + + +def ToAbsolute(path): + if not path: + return path + return os.path.expandvars(os.path.expanduser(path)) + + +def DeleteFile(file_pattern): + DeleteFileFunc(file_pattern, lambda item: os.remove(item)) + + +def SecureDeleteFile(file_pattern): + DeleteFileFunc(file_pattern, lambda item: Run(['shred', '--remove', '--zero', item])) + + +def DeleteFileFunc(file_pattern, delete_func): + items = glob.glob(ToAbsolute(file_pattern)) + for item in items: + logging.warning('Deleting %s', item) + delete_func(item) + + +def DeleteDirectory(dir_path): + shutil.rmtree(dir_path) + + +def CreateTempDirectory(base_dir=None): + return tempfile.mkdtemp(dir=ToAbsolute(base_dir), prefix='gcearch') + + +def WriteFile(path, content): + with open(ToAbsolute(path), 'w') as fp: + fp.write(content) + + +def AppendFile(path, content): + with open(ToAbsolute(path), 'a') as fp: + fp.write(content) + + +def RunChroot(base_dir, command, use_custom_path=True): + base_dir = ToAbsolute(base_dir) + if use_custom_path: + chroot_file = os.path.join(base_dir, 'bin/arch-chroot') + else: + chroot_file = 'arch-chroot' + SudoRun([chroot_file, base_dir, '/bin/bash', '-c', command]) + + +def CopyFiles(source_pattern, dest): + """Copies a set of files based on glob pattern to a directory. + Avoiding shutil.copyfile because of bugs.python.org/issue10016.""" + items = glob.glob(ToAbsolute(source_pattern)) + for item in items: + Run(['cp', '-Rf', item, dest]) + + +def CopyBuilder(base_dir): + script_dir = os.path.dirname(os.path.realpath(__file__)) + temp_dir = CreateTempDirectory(base_dir=base_dir) + DeleteDirectory(temp_dir) + relative_dir = os.path.relpath(temp_dir, base_dir) + shutil.copytree(script_dir, temp_dir, ignore=shutil.ignore_patterns('*.tar.gz', '*.raw')) + return relative_dir + + +def EncodeArgs(decoded_args): + return base64.standard_b64encode(gzip.compress(bytes(json.dumps(decoded_args), 'utf-8'))).decode('utf-8') + + +def DecodeArgs(encoded_args): + return json.loads(gzip.decompress(base64.standard_b64decode(encoded_args)).decode('utf-8')) + + +def DebugBash(): + Run(['/bin/bash']) + + +def DebugPrintFile(file_path): + logging.info('==================================================================================') + logging.info('File: %s', file_path) + logging.info('==================================================================================') + Run(['cat', file_path]) + + +def Sync(): + Run(['sync']) + + +def EnableService(service_name): + Run(['systemctl', 'enable', service_name]) + + +def Symlink(source_file, dest_file): + Run(['ln', '-s', source_file, dest_file]) + + +def AurInstall(name=None, pkbuild_url=None): + if name: + pkbuild_url = 'https://aur.archlinux.org/packages/%s/%s/PKGBUILD' % (name.lower()[:2], name.lower()) + workspace_dir = CreateTempDirectory() + DownloadFile(pkbuild_url, os.path.join(workspace_dir, 'PKGBUILD')) + Run(['makepkg', '--asroot'], cwd=workspace_dir) + tarball = glob.glob(os.path.join(workspace_dir, '*.tar*')) + tarball = tarball[0] + Pacman(['-U', tarball], cwd=workspace_dir) + return tarball + + +def Pacstrap(base_dir, params): + Run(['pacstrap', base_dir] + params) + + +def Pacman(params, cwd=None): + #, '--debug' + Run(['pacman', '--noconfirm'] + params, cwd=cwd) + + +def InstallPackages(package_list): + Pacman(['-S'] + package_list) + + +def SetupArchLocale(): + AppendFile('/etc/locale.gen', ETC_LOCALE_GEN) + Run(['locale-gen']) + Run(['localectl', 'set-locale', 'LANG="en_US.UTF-8"', 'LC_COLLATE="C"']) + + +class ImageMapper(object): + """Interface for kpartx, mount, and umount.""" + def __init__(self, raw_disk, mount_path): + self._raw_disk = raw_disk + self._mount_path = mount_path + self._device_map = None + self._mount_points = None + + def _LoadPartitionsIfNeeded(self): + if not self._device_map: + self.LoadPartitions() + + def LoadPartitions(self): + return_code, out, err = SudoRun(['kpartx', '-l', self._raw_disk], capture_output=True) + # Expected Format + # loop2p1 : 0 10483712 /dev/loop2 2048 + # loop2p2 : 0 1 /dev/loop2 2047 + # loop deleted : /dev/loop2 + + self._device_map = {} + lines = out.splitlines() + for line in lines: + parts = str(line).split() + if len(parts) == 6: + mapping = { + 'name': parts[0], + 'size_blocks': parts[3], + 'parent': parts[4], + 'start_block': parts[5], + 'path': '/dev/mapper/%s' % str(parts[0]) + } + logging.info('Mapping: %s', mapping) + self._device_map[mapping['name']] = mapping + if len(self._device_map) == 1: + self._mount_points = [self._mount_path] + else: + self._mount_points = [] + for name in list(self._device_map.keys()): + self._mount_points.append(os.path.join(self._mount_path, name)) + + def ForEachDevice(self, func): + for name in list(self._device_map.keys()): + spec = self._device_map[name] + func(spec) + + def ForEachDeviceWithIndex(self, func): + i = 0 + for name in list(self._device_map.keys()): + spec = self._device_map[name] + func(i, spec) + i += 1 + + def GetFirstMapping(self): + logging.info('DeviceMap: %s', self.GetDeviceMap()) + return next(iter(self.GetDeviceMap().values())) + + def GetDeviceMap(self): + return self._device_map + + def Sync(self): + Run(['sync']) + + def Map(self): + SudoRun(['kpartx', '-a', '-v', '-s', self._raw_disk]) + self.LoadPartitions() + + def Unmap(self): + self.Sync() + time.sleep(2) + SudoRun(['kpartx', '-d', '-v', '-s', self._raw_disk]) + self._device_map = None + + def Mount(self): + self._LoadPartitionsIfNeeded() + self._ThrowIfBadMountMap() + def MountCmd(index, spec): + mount_point = self._mount_points[index] + CreateDirectory(mount_point) + SudoRun(['mount', spec['path'], mount_point]) + self.ForEachDeviceWithIndex(MountCmd) + + def Unmount(self): + self._LoadPartitionsIfNeeded() + self._ThrowIfBadMountMap() + for path in self._mount_points: + SudoRun(['umount', path]) + self.Sync() + time.sleep(2) + + def _ThrowIfBadMountMap(self): + if not self._mount_points: + raise IOError('Attempted to found {0} without a mount points.'.format(self._raw_disk)) + if len(self._mount_points) != len(list(self._device_map.keys())): + raise IOError('Number of device maps ({0}) does not match mount points ({1}).'.format( + len(list(self._device_map.keys())), len(self._mount_points) + )) |