#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Utility script to create a pypi-like directory structure (localpi)
from a number of Python packages in a directory of the local filesystem.
DIRECTORY STRUCTURE (before):
+-- downloads/
+-- alice-1.0.zip
+-- alice-1.0.tar.gz
+-- bob-1.3.0.tar.gz
+-- bob-1.4.2.tar.gz
+-- charly-1.0.tar.bz2
DIRECTORY STRUCTURE (afterwards):
+-- downloads/
+-- simple/
| +-- alice/index.html --> ../../alice-*.*
| +-- bob/index.html --> ../../bob-*.*
| +-- charly/index.html --> ../../charly-*.*
| +-- index.html --> alice/, bob/, ...
+-- alice-1.0.zip
+-- alice-1.0.tar.gz
+-- bob-1.3.0.tar.gz
+-- bob-1.4.2.tar.gz
+-- charly-1.0.tar.bz2
USAGE EXAMPLE:
mkdir -p /tmp/downloads
pip install --download=/tmp/downloads argparse Jinja2
make_localpi.py /tmp/downloads
pip install --index-url=file:///tmp/downloads/simple argparse Jinja2
ALTERNATIVE:
pip install --download=/tmp/downloads argparse Jinja2
pip install --find-links=/tmp/downloads --no-index argparse Jinja2
"""
from __future__ import with_statement, print_function
from fnmatch import fnmatch
import os.path
import shutil
import sys
__author__ = "Jens Engel"
__version__ = "0.2"
__license__ = "BSD"
__copyright__ = "(c) 2013 by Jens Engel"
class Package(object):
"""
Package entity that keeps track of:
* one or more versions of this package
* one or more archive types
"""
PATTERNS = [
"*.egg", "*.exe", "*.whl", "*.zip", "*.tar.gz", "*.tar.bz2", "*.7z"
]
def __init__(self, filename, name=None):
if not name and filename:
name = self.get_pkgname(filename)
self.name = name
self.files = []
if filename:
self.files.append(filename)
@property
def versions(self):
versions_info = [ self.get_pkgversion(p) for p in self.files ]
return versions_info
@classmethod
def get_pkgversion(cls, filename):
parts = os.path.basename(filename).rsplit("-", 1)
version = ""
if len(parts) >= 2:
version = parts[1]
for pattern in cls.PATTERNS:
assert pattern.startswith("*")
suffix = pattern[1:]
if version.endswith(suffix):
version = version[:-len(suffix)]
break
return version
@staticmethod
def get_pkgname(filename):
name = os.path.basename(filename).rsplit("-", 1)[0]
if name.startswith("http%3A") or name.startswith("https%3A"):
# -- PIP DOWNLOAD-CACHE PACKAGE FILE NAME SCHEMA:
pos = name.rfind("%2F")
name = name[pos+3:]
return name
@staticmethod
def splitext(filename):
fname = os.path.splitext(filename)[0]
if fname.endswith(".tar"):
fname = os.path.splitext(fname)[0]
return fname
@classmethod
def isa(cls, filename):
basename = os.path.basename(filename)
if basename.startswith("."):
return False
for pattern in cls.PATTERNS:
if fnmatch(filename, pattern):
return True
return False
def make_index_for(package, index_dir, verbose=True):
"""
Create an 'index.html' for one package.
:param package: Package object to use.
:param index_dir: Where 'index.html' should be created.
"""
index_template = """\
{title}
{title}
"""
item_template = '{0}'
index_filename = os.path.join(index_dir, "index.html")
if not os.path.isdir(index_dir):
os.makedirs(index_dir)
parts = []
for pkg_filename in package.files:
pkg_name = os.path.basename(pkg_filename)
if pkg_name == "index.html":
# -- ROOT-INDEX:
pkg_name = os.path.basename(os.path.dirname(pkg_filename))
else:
pkg_name = package.splitext(pkg_name)
pkg_relpath_to = os.path.relpath(pkg_filename, index_dir)
parts.append(item_template.format(pkg_name, pkg_relpath_to))
if not parts:
print("OOPS: Package %s has no files" % package.name)
return
if verbose:
root_index = not Package.isa(package.files[0])
if root_index:
info = "with %d package(s)" % len(package.files)
else:
package_versions = sorted(set(package.versions))
info = ", ".join(reversed(package_versions))
message = "%-30s %s" % (package.name, info)
print(message)
with open(index_filename, "w") as f:
packages = "\n".join(parts)
text = index_template.format(title=package.name, packages=packages)
f.write(text.strip())
f.close()
def make_package_index(download_dir):
"""
Create a pypi server like file structure below download directory.
:param download_dir: Download directory with packages.
EXAMPLE BEFORE:
+-- downloads/
+-- alice-1.0.zip
+-- alice-1.0.tar.gz
+-- bob-1.3.0.tar.gz
+-- bob-1.4.2.tar.gz
+-- charly-1.0.tar.bz2
EXAMPLE AFTERWARDS:
+-- downloads/
+-- simple/
| +-- alice/index.html --> ../../alice-*.*
| +-- bob/index.html --> ../../bob-*.*
| +-- charly/index.html --> ../../charly-*.*
| +-- index.html --> alice/index.html, bob/index.html, ...
+-- alice-1.0.zip
+-- alice-1.0.tar.gz
+-- bob-1.3.0.tar.gz
+-- bob-1.4.2.tar.gz
+-- charly-1.0.tar.bz2
"""
if not os.path.isdir(download_dir):
raise ValueError("No such directory: %r" % download_dir)
pkg_rootdir = os.path.join(download_dir, "simple")
if os.path.isdir(pkg_rootdir):
shutil.rmtree(pkg_rootdir, ignore_errors=True)
os.mkdir(pkg_rootdir)
# -- STEP: Collect all packages.
package_map = {}
packages = []
for filename in sorted(os.listdir(download_dir)):
if not Package.isa(filename):
continue
pkg_filepath = os.path.join(download_dir, filename)
package_name = Package.get_pkgname(pkg_filepath)
package = package_map.get(package_name, None)
if not package:
# -- NEW PACKAGE DETECTED: Store/register package.
package = Package(pkg_filepath)
package_map[package.name] = package
packages.append(package)
else:
# -- SAME PACKAGE: Collect other variant/version.
package.files.append(pkg_filepath)
# -- STEP: Make local PYTHON PACKAGE INDEX.
root_package = Package(None, "Python Package Index")
root_package.files = [ os.path.join(pkg_rootdir, pkg.name, "index.html")
for pkg in packages ]
make_index_for(root_package, pkg_rootdir)
for package in packages:
index_dir = os.path.join(pkg_rootdir, package.name)
make_index_for(package, index_dir)
# -----------------------------------------------------------------------------
# MAIN:
# -----------------------------------------------------------------------------
if __name__ == "__main__":
if (len(sys.argv) != 2) or "-h" in sys.argv[1:] or "--help" in sys.argv[1:]:
print("USAGE: %s DOWNLOAD_DIR" % os.path.basename(sys.argv[0]))
print(__doc__)
sys.exit(1)
make_package_index(sys.argv[1])