support/testing: Add download tests for SCP/SFTP

Add download test infrastructure which starts an OpenSSH server using
the sshd binary installed on the Buildroot host. This server can then be
used to test the expected usage of the SCP and SFTP download methods.
The test creates new SSH keys for the server and client, so that the
server can be run as a non-root user.

A new test module has been added called `tests.download.sshd` which
contains helper methods to create the SSH keys and a class called
`OpenSSHDaemon` which handles the sshd server component.

The tests download example packages in the br2-external project `ssh`.
They check the following conditions for both SCP and SFTP download
methods:
- Correct hash.
- Incorrect hash.
- No hash file.

The SSH download test infrastructure is based on test_git.py.

Signed-off-by: Thomas Preston <thomas.preston@codethink.co.uk>
[Arnout:
 - remove spurious end-of-line backslash;
 - remove unnecessary executable bit;
 - skip test instead of failing if sshd, ssh-keygen, scp or sftp are not
   found;
 - decode the output of subprocess;
 - use subprocess.check_output instead of subprocess.get_output;
 - use subprocess.check_call instead of manually checking return code;
 - don't set always-overridden SSHD_PORT_NUMBER in .mk file;
 - explicitly set sshd options on commandline instead of relying on host
   /etc/sshd/sshd_config;
 - let sshd listen only on localhost;
 - user internal sftp server;
 - disable BACKUP_SITE, no network is supposed to be accessed;
 - remove the -bad and -nohash versions;
 - rename {sftp,scp}-good to plain {sftp,scp};
 - move the sftp and scp packages into a single "ssh" external.
]
Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>

support/testing/tests/download/br2-external/ssh/external.desc

@@ -0,0 +1 @@
+name: SSH

support/testing/tests/download/br2-external/ssh/external.mk

@@ -0,0 +1 @@
+include $(sort $(wildcard $(BR2_EXTERNAL_SSH_PATH)/package/*/*.mk))

support/testing/tests/download/br2-external/ssh/package/scp/scp.hash

@@ -0,0 +1 @@
+sha256  b457c1a37ba7405e8806b93f3d5cc82165db0b0cad25d203f112e32c7a30c0be  ssh-test-1.0.tar.xz

support/testing/tests/download/br2-external/ssh/package/scp/scp.mk

@@ -0,0 +1,17 @@
+################################################################################
+#
+# scp
+#
+################################################################################
+
+SCP_VERSION = 1.0
+SCP_SOURCE = ssh-test-$(SCP_VERSION).tar.xz
+SCP_SITE = scp://localhost:$(SSHD_TEST_DIR)
+SCP_DL_OPTS = \
+	-P $(SSHD_PORT_NUMBER) \
+	-i $(SSH_IDENTITY) \
+	-o "UserKnownHostsFile=/dev/null" \
+	-o "StrictHostKeyChecking=no" \
+	-o "CheckHostIP=no"
+
+$(eval $(generic-package))

support/testing/tests/download/br2-external/ssh/package/sftp/sftp.hash

@@ -0,0 +1 @@
+sha256  b457c1a37ba7405e8806b93f3d5cc82165db0b0cad25d203f112e32c7a30c0be  ssh-test-1.0.tar.xz

support/testing/tests/download/br2-external/ssh/package/sftp/sftp.mk

@@ -0,0 +1,17 @@
+################################################################################
+#
+# sftp
+#
+################################################################################
+
+SFTP_VERSION = 1.0
+SFTP_SOURCE = ssh-test-$(SFTP_VERSION).tar.xz
+SFTP_SITE = sftp://localhost/$(SSHD_TEST_DIR)
+SFTP_DL_OPTS = \
+	-P $(SSHD_PORT_NUMBER) \
+	-i $(SSH_IDENTITY) \
+	-o "UserKnownHostsFile=/dev/null" \
+	-o "StrictHostKeyChecking=no" \
+	-o "CheckHostIP=no"
+
+$(eval $(generic-package))

support/testing/tests/download/sshd.py

@@ -0,0 +1,149 @@
+import os
+import shutil
+import subprocess
+from unittest import SkipTest
+
+# subprocess does not kill the child daemon when a test case fails by raising
+# an exception. So use pexpect instead.
+import pexpect
+
+import infra
+
+
+SSHD_PORT_INITIAL = 2222
+SSHD_PORT_LAST = SSHD_PORT_INITIAL + 99
+SSHD_PATH = "/usr/sbin/sshd"
+SSHD_HOST_DIR = "host"
+
+# SSHD_KEY_DIR is where the /etc/ssh/ssh_host_*_key files go
+SSHD_KEY_DIR = os.path.join(SSHD_HOST_DIR, "etc/ssh")
+SSHD_KEY = os.path.join(SSHD_KEY_DIR, "ssh_host_ed25519_key")
+
+# SSH_CLIENT_KEY_DIR is where the client id_rsa key and authorized_keys files go
+SSH_CLIENT_KEY_DIR = os.path.join(SSHD_HOST_DIR, "home/br-user/ssh")
+SSH_CLIENT_KEY = os.path.join(SSH_CLIENT_KEY_DIR, "id_rsa")
+SSH_AUTH_KEYS_FILE = os.path.join(SSH_CLIENT_KEY_DIR, "authorized_keys")
+
+
+class OpenSSHDaemon():
+
+    def __init__(self, builddir, logtofile):
+        """
+        Start an OpenSSH SSH Daemon
+
+        In order to support test cases in parallel, select the port the
+        server will listen to in runtime. Since there is no reliable way
+        to allocate the port prior to starting the server (another
+        process in the host machine can use the port between it is
+        selected from a list and it is really allocated to the server)
+        try to start the server in a port and in the case it is already
+        in use, try the next one in the allowed range.
+        """
+        self.daemon = None
+        self.port = None
+
+        self.logfile = infra.open_log_file(builddir, "sshd", logtofile)
+
+        server_keyfile = os.path.join(builddir, SSHD_KEY)
+        auth_keys_file = os.path.join(builddir, SSH_AUTH_KEYS_FILE)
+        daemon_cmd = [SSHD_PATH,
+                      "-D", # or use -ddd to debug
+                      "-e",
+                      "-h", server_keyfile,
+                      "-f", "/dev/null",
+                      "-o", "ListenAddress=localhost",
+                      "-o", "PidFile=none",
+                      "-o", "AuthenticationMethods=publickey",
+                      "-o", "StrictModes=no",
+                      "-o", "Subsystem=sftp internal-sftp",
+                      "-o", "AuthorizedKeysFile={}".format(auth_keys_file)]
+        for port in range(SSHD_PORT_INITIAL, SSHD_PORT_LAST + 1):
+            cmd = daemon_cmd + ["-p", "{}".format(port)]
+            self.logfile.write(
+                "> starting sshd with '{}'\n".format(" ".join(cmd)))
+            try:
+                self.daemon = pexpect.spawn(cmd[0], cmd[1:], logfile=self.logfile,
+                                            encoding='utf-8')
+            except pexpect.exceptions.ExceptionPexpect as e:
+                self.logfile.write("> {} - skipping\n".format(e))
+                raise SkipTest(str(e))
+
+            ret = self.daemon.expect([
+                # Success
+                "Server listening on .* port {}.".format(port),
+                # Failure
+                "Cannot bind any address."])
+            if ret == 0:
+                self.port = port
+                return
+        raise SystemError("Could not find a free port to run sshd")
+
+    def stop(self):
+        if self.daemon is None:
+            return
+        self.daemon.terminate(force=True)
+
+
+def generate_keys_server(builddir, logfile):
+    """Generate keys required to run an OpenSSH Daemon."""
+    keyfile = os.path.join(builddir, SSHD_KEY)
+    if os.path.exists(keyfile):
+        logfile.write("> SSH server key already exists '{}'".format(keyfile))
+        return
+
+    hostdir = os.path.join(builddir, SSHD_HOST_DIR)
+    keydir = os.path.join(builddir, SSHD_KEY_DIR)
+    os.makedirs(hostdir, exist_ok=True)
+    os.makedirs(keydir, exist_ok=True)
+
+    cmd = ["ssh-keygen", "-A", "-f", hostdir]
+    logfile.write(
+        "> generating SSH server keys with '{}'\n".format(" ".join(cmd)))
+    # When ssh-keygen fails to create an SSH server key it doesn't return a
+    # useful error code. So use check for an error message in the output
+    # instead.
+    try:
+        out = subprocess.check_output(cmd, encoding='utf-8')
+    except FileNotFoundError:
+        logfile.write("> ssh-keygen not found - skipping\n")
+        raise SkipTest("ssh-keygen not found")
+
+    logfile.write(out)
+    if "Could not save your public key" in out:
+        raise SystemError("Could not generate SSH server keys")
+
+
+def generate_keys_client(builddir, logfile):
+    """Generate keys required to log into an OpenSSH Daemon via SCP or SFTP."""
+    keyfile = os.path.join(builddir, SSH_CLIENT_KEY)
+    if os.path.exists(keyfile):
+        logfile.write("> SSH client key already exists '{}'".format(keyfile))
+        return
+
+    keydir = os.path.join(builddir, SSH_CLIENT_KEY_DIR)
+    os.makedirs(keydir, exist_ok=True)
+
+    cmd = ["ssh-keygen",
+           "-f", keyfile,
+           "-b", "2048",
+           "-t", "rsa",
+           "-N", "",
+           "-q"]
+    logfile.write(
+        "> generating SSH client keys with '{}'\n".format(" ".join(cmd)))
+    try:
+        subprocess.check_call(cmd, stdout=logfile, stderr=logfile)
+    except FileNotFoundError:
+        logfile.write("> ssh-keygen not found - skipping\n")
+        raise SkipTest("ssh-keygen not found")
+
+    # Allow key-based login for this user (so that we can fetch from localhost)
+    pubkeyfile = os.path.join(keydir, "{}.pub".format(keyfile))
+    authfile = os.path.join(keydir, "authorized_keys")
+    shutil.copy(pubkeyfile, authfile)
+
+
+def generate_keys(builddir, logtofile):
+    logfile = infra.open_log_file(builddir, "ssh-keygen", logtofile)
+    generate_keys_server(builddir, logfile)
+    generate_keys_client(builddir, logfile)

support/testing/tests/download/test_ssh.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+
+import tests.download.sshd
+
+import infra
+
+
+class SSHTestBase(infra.basetest.BRConfigTest):
+    config = infra.basetest.MINIMAL_CONFIG + '''
+BR2_BACKUP_SITE=""
+'''
+    sshd_test_dir = infra.filepath("tests/download/sshd")
+    sshd = None
+
+    def setUp(self):
+        super(SSHTestBase, self).setUp()
+
+        self.show_msg("Generating keys")
+        tests.download.sshd.generate_keys(self.builddir, self.logtofile)
+
+        self.show_msg("Starting sshd")
+        self.sshd = tests.download.sshd.OpenSSHDaemon(self.builddir,
+                                                      self.logtofile)
+
+    def tearDown(self):
+        self.show_msg("Stopping sshd")
+        if self.sshd:
+            self.sshd.stop()
+        super(SSHTestBase, self).tearDown()
+
+    def download_package(self, package):
+        self.show_msg("Downloading {}".format(package))
+        # store downloaded tarball inside the output dir so the test infra
+        # cleans it up at the end
+        dl_dir = os.path.join(self.builddir, "dl")
+        ssh_identity = os.path.join(self.builddir,
+                                    tests.download.sshd.SSH_CLIENT_KEY)
+        # enforce that we test the download
+        if os.path.exists(dl_dir):
+            shutil.rmtree(dl_dir)
+        env = {"BR2_DL_DIR": dl_dir,
+               "SSHD_PORT_NUMBER": str(self.sshd.port),
+               "SSHD_TEST_DIR": self.sshd_test_dir,
+               "SSH_IDENTITY": ssh_identity}
+        self.b.build(["{}-dirclean".format(package),
+                      "{}-source".format(package)],
+                     env)
+
+
+class TestSCP(SSHTestBase):
+    br2_external = [infra.filepath("tests/download/br2-external/ssh")]
+
+    def test_run(self):
+        self.download_package("scp")
+
+
+class TestSFTP(SSHTestBase):
+    br2_external = [infra.filepath("tests/download/br2-external/ssh")]
+
+    def test_run(self):
+        self.download_package("sftp")