Merge pull request #3 from gchanan/testing_the_thing

[TESTING THE THING] Testing the thing

Author: gchanan

.circleci/.gitignore

@@ -0,0 +1,2 @@
+*.svg
+*.png

.circleci/README.md

@@ -0,0 +1,171 @@
+"""
+This module models the tree of configuration variants
+for "smoketest" builds.
+
+Each subclass of ConfigNode represents a layer of the configuration hierarchy.
+These tree nodes encapsulate the logic for whether a branch of the hierarchy
+should be "pruned".
+
+In addition to generating config.yml content, the tree is also traversed
+to produce a visualization of config dimensions.
+"""
+
+from collections import OrderedDict
+
+from cimodel.lib.conf_tree import ConfigNode
+import cimodel.data.dimensions as dimensions
+
+
+LINKING_DIMENSIONS = [
+ "shared",
+ "static",
+]
+
+
+DEPS_INCLUSION_DIMENSIONS = [
+ "with-deps",
+ "without-deps",
+]
+
+
+def get_processor_arch_name(cuda_version):
+ return "cpu" if not cuda_version else "cu" + cuda_version
+
+
+LINUX_PACKAGE_VARIANTS = OrderedDict(
+ manywheel=[
+ "2.7m",
+ "2.7mu",
+ "3.5m",
+ "3.6m",
+ "3.7m",
+ ],
+ conda=dimensions.STANDARD_PYTHON_VERSIONS,
+ libtorch=[
+ "2.7m",
+ ],
+)
+
+CONFIG_TREE_DATA = OrderedDict(
+ linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
+ macos=([None], OrderedDict(
+ wheel=dimensions.STANDARD_PYTHON_VERSIONS,
+ conda=dimensions.STANDARD_PYTHON_VERSIONS,
+ libtorch=[
+ "2.7",
+ ],
+ )),
+)
+
+# GCC config variants:
+#
+# All the nightlies (except libtorch with new gcc ABI) are built with devtoolset7,
+# which can only build with old gcc ABI. It is better than devtoolset3
+# because it understands avx512, which is needed for good fbgemm performance.
+#
+# Libtorch with new gcc ABI is built with gcc 5.4 on Ubuntu 16.04.
+LINUX_GCC_CONFIG_VARIANTS = OrderedDict(
+ manywheel=['devtoolset7'],
+ conda=['devtoolset7'],
+ libtorch=[
+ "devtoolset7",
+ "gcc5.4_cxx11-abi",
+ ],
+)
+
+
+class TopLevelNode(ConfigNode):
+ def __init__(self, node_name, config_tree_data, smoke):
+ super(TopLevelNode, self).__init__(None, node_name)
+
+ self.config_tree_data = config_tree_data
+ self.props["smoke"] = smoke
+
+ def get_children(self):
+ return [OSConfigNode(self, x, c, p) for (x, (c, p)) in self.config_tree_data.items()]
+
+
+class OSConfigNode(ConfigNode):
+ def __init__(self, parent, os_name, cuda_versions, py_tree):
+ super(OSConfigNode, self).__init__(parent, os_name)
+
+ self.py_tree = py_tree
+ self.props["os_name"] = os_name
+ self.props["cuda_versions"] = cuda_versions
+
+ def get_children(self):
+ return [PackageFormatConfigNode(self, k, v) for k, v in self.py_tree.items()]
+
+
+class PackageFormatConfigNode(ConfigNode):
+ def __init__(self, parent, package_format, python_versions):
+ super(PackageFormatConfigNode, self).__init__(parent, package_format)
+
+ self.props["python_versions"] = python_versions
+ self.props["package_format"] = package_format
+
+ def get_children(self):
+ if self.find_prop("os_name") == "linux":
+ return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]]
+ else:
+ return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+
+
+class LinuxGccConfigNode(ConfigNode):
+ def __init__(self, parent, gcc_config_variant):
+ super(LinuxGccConfigNode, self).__init__(parent, "GCC_CONFIG_VARIANT=" + str(gcc_config_variant))
+
+ self.props["gcc_config_variant"] = gcc_config_variant
+
+ def get_children(self):
+ cuda_versions = self.find_prop("cuda_versions")
+
+ # XXX devtoolset7 on CUDA 9.0 is temporarily disabled
+ # see https://github.com/pytorch/pytorch/issues/20066
+ if self.find_prop("gcc_config_variant") == 'devtoolset7':
+ cuda_versions = filter(lambda x: x != "90", cuda_versions)
+
+ return [ArchConfigNode(self, v) for v in cuda_versions]
+
+
+class ArchConfigNode(ConfigNode):
+ def __init__(self, parent, cu):
+ super(ArchConfigNode, self).__init__(parent, get_processor_arch_name(cu))
+
+ self.props["cu"] = cu
+
+ def get_children(self):
+ return [PyVersionConfigNode(self, v) for v in self.find_prop("python_versions")]
+
+
+class PyVersionConfigNode(ConfigNode):
+ def __init__(self, parent, pyver):
+ super(PyVersionConfigNode, self).__init__(parent, pyver)
+
+ self.props["pyver"] = pyver
+
+ def get_children(self):
+
+ smoke = self.find_prop("smoke")
+ package_format = self.find_prop("package_format")
+ os_name = self.find_prop("os_name")
+
+ has_libtorch_variants = package_format == "libtorch" and os_name == "linux"
+ linking_variants = LINKING_DIMENSIONS if has_libtorch_variants else []
+
+ return [LinkingVariantConfigNode(self, v) for v in linking_variants]
+
+
+class LinkingVariantConfigNode(ConfigNode):
+ def __init__(self, parent, linking_variant):
+ super(LinkingVariantConfigNode, self).__init__(parent, linking_variant)
+
+ def get_children(self):
+ return [DependencyInclusionConfigNode(self, v) for v in DEPS_INCLUSION_DIMENSIONS]
+
+
+class DependencyInclusionConfigNode(ConfigNode):
+ def __init__(self, parent, deps_variant):
+ super(DependencyInclusionConfigNode, self).__init__(parent, deps_variant)
+
+ self.props["libtorch_variant"] = "-".join([self.parent.get_label(), self.get_label()])

torch/nn/_functions/__init__.py .circleci/cimodel/__init__.pytorch/nn/_functions/__init__.py renamed to .circleci/cimodel/__init__.py

@@ -0,0 +1,167 @@
+from collections import OrderedDict
+
+import cimodel.data.binary_build_data as binary_build_data
+import cimodel.lib.conf_tree as conf_tree
+import cimodel.lib.miniutils as miniutils
+
+
+class Conf(object):
+ def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant):
+
+ self.os = os
+ self.cuda_version = cuda_version
+ self.pydistro = pydistro
+ self.parms = parms
+ self.smoke = smoke
+ self.libtorch_variant = libtorch_variant
+ self.gcc_config_variant = gcc_config_variant
+
+ def gen_build_env_parms(self):
+ elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
+ if self.gcc_config_variant is not None:
+ elems.append(str(self.gcc_config_variant))
+ return elems
+
+ def gen_docker_image(self):
+ if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
+ return miniutils.quote("pytorch/pytorch-binary-docker-image-ubuntu16.04:latest")
+
+ docker_word_substitution = {
+ "manywheel": "manylinux",
+ "libtorch": "manylinux",
+ }
+
+ docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
+
+ # The cpu nightlies are built on the pytorch/manylinux-cuda100 docker image
+ alt_docker_suffix = self.cuda_version or "100"
+ docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
+ return miniutils.quote("pytorch/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
+
+ def get_name_prefix(self):
+ return "smoke" if self.smoke else "binary"
+
+ def gen_build_name(self, build_or_test, nightly):
+
+ parts = [self.get_name_prefix(), self.os] + self.gen_build_env_parms()
+
+ if nightly:
+ parts.append("nightly")
+
+ if self.libtorch_variant:
+ parts.append(self.libtorch_variant)
+
+ if not self.smoke:
+ parts.append(build_or_test)
+
+ joined = "_".join(parts)
+ return joined.replace(".", "_")
+
+ def gen_workflow_job(self, phase, upload_phase_dependency=None, nightly=False):
+ job_def = OrderedDict()
+ job_def["name"] = self.gen_build_name(phase, nightly)
+ job_def["build_environment"] = miniutils.quote(" ".join(self.gen_build_env_parms()))
+ job_def["requires"] = ["setup"]
+ if self.smoke:
+ job_def["requires"].append("update_s3_htmls_for_nightlies")
+ job_def["requires"].append("update_s3_htmls_for_nightlies_devtoolset7")
+ job_def["filters"] = {"branches": {"only": "postnightly"}}
+ else:
+ job_def["filters"] = {"branches": {"only": "nightly"}}
+ if self.libtorch_variant:
+ job_def["libtorch_variant"] = miniutils.quote(self.libtorch_variant)
+ if phase == "test":
+ if not self.smoke:
+ job_def["requires"].append(self.gen_build_name("build", nightly))
+ if not (self.smoke and self.os == "macos"):
+ job_def["docker_image"] = self.gen_docker_image()
+
+ if self.cuda_version:
+ job_def["use_cuda_docker_runtime"] = miniutils.quote("1")
+ else:
+ if self.os == "linux" and phase != "upload":
+ job_def["docker_image"] = self.gen_docker_image()
+
+ if phase == "test":
+ if self.cuda_version:
+ job_def["resource_class"] = "gpu.medium"
+ if phase == "upload":
+ job_def["context"] = "org-member"
+ job_def["requires"] = ["setup", self.gen_build_name(upload_phase_dependency, nightly)]
+
+ os_name = miniutils.override(self.os, {"macos": "mac"})
+ job_name = "_".join([self.get_name_prefix(), os_name, phase])
+ return {job_name : job_def}
+
+def get_root(smoke, name):
+
+ return binary_build_data.TopLevelNode(
+ name,
+ binary_build_data.CONFIG_TREE_DATA,
+ smoke,
+ )
+
+
+def gen_build_env_list(smoke):
+
+ root = get_root(smoke, "N/A")
+ config_list = conf_tree.dfs(root)
+
+ newlist = []
+ for c in config_list:
+ conf = Conf(
+ c.find_prop("os_name"),
+ c.find_prop("cu"),
+ c.find_prop("package_format"),
+ [c.find_prop("pyver")],
+ c.find_prop("smoke"),
+ c.find_prop("libtorch_variant"),
+ c.find_prop("gcc_config_variant"),
+ )
+ newlist.append(conf)
+
+ return newlist
+
+
+def predicate_exclude_nonlinux_and_libtorch(config):
+ return config.os == "linux"
+
+
+def get_nightly_uploads():
+ configs = gen_build_env_list(False)
+ mylist = []
+ for conf in configs:
+ phase_dependency = "test" if predicate_exclude_nonlinux_and_libtorch(conf) else "build"
+ mylist.append(conf.gen_workflow_job("upload", phase_dependency, nightly=True))
+
+ return mylist
+
+def get_nightly_tests():
+
+ configs = gen_build_env_list(False)
+ filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
+
+ tests = []
+ for conf_options in filtered_configs:
+ yaml_item = conf_options.gen_workflow_job("test", nightly=True)
+ tests.append(yaml_item)
+
+ return tests
+
+
+def get_jobs(toplevel_key, smoke):
+ jobs_list = []
+ configs = gen_build_env_list(smoke)
+ phase = "build" if toplevel_key == "binarybuilds" else "test"
+ for build_config in configs:
+ jobs_list.append(build_config.gen_workflow_job(phase, nightly=True))
+
+ return jobs_list
+
+
+def get_binary_build_jobs():
+ return get_jobs("binarybuilds", False)
+
+
+def get_binary_smoke_test_jobs():
+ return get_jobs("binarysmoketests", True)