This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "IPFire 3.x development tree".
The branch, master has been updated via ebb540ea95064456f29c5c2930000219c5f9d5ea (commit) via 8c05c4eb35663512db898f8e78563966d8a6d24d (commit) via b7d904b21f6342f4239bb3892f776858e5b5420e (commit) via a45613f6c193784f6674af3d18a95e8f495da8c8 (commit) via 96765864099cbc227112815e4d12ee2c1df31810 (commit) via 2f129e1518be27d799134787e36d040016fa15de (commit) via 42e0b48fa349ed9b3673a6212dd8e5b89572f764 (commit) via 4677ce9eba457bfeb426ab8d54abd284223a0abc (commit) via 340c29e78480b0301bc87a4c4f491af80cd81c03 (commit) via e54b336efab72012377bbea49b6b92615440649c (commit) from f28f3f056c879081c2043e76e5ac9ff314c68fda (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit ebb540ea95064456f29c5c2930000219c5f9d5ea Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 16:43:41 2010 +0100
naoki: Fix toolchain build.
commit 8c05c4eb35663512db898f8e78563966d8a6d24d Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 16:43:18 2010 +0100
naoki: Add checks if a package was already built.
commit b7d904b21f6342f4239bb3892f776858e5b5420e Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 16:42:03 2010 +0100
naoki: Fix urlgrabber progress bar.
commit a45613f6c193784f6674af3d18a95e8f495da8c8 Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 16:31:48 2010 +0100
naoki: Export PKG_ARCH to environment.
commit 96765864099cbc227112815e4d12ee2c1df31810 Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 14:23:39 2010 +0100
naoki: Changed naming of some variables.
commit 2f129e1518be27d799134787e36d040016fa15de Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 14:23:18 2010 +0100
naoki: Use right CFLAGS for given arch.
commit 42e0b48fa349ed9b3673a6212dd8e5b89572f764 Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 11:25:53 2010 +0100
naoki: Calculate a better number of concurrent processes.
commit 4677ce9eba457bfeb426ab8d54abd284223a0abc Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 10:57:40 2010 +0100
naoki: Add arch information to package name.
commit 340c29e78480b0301bc87a4c4f491af80cd81c03 Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 10:38:35 2010 +0100
naoki: Apply -a, --arch command line setting.
commit e54b336efab72012377bbea49b6b92615440649c Author: Michael Tremer michael.tremer@ipfire.org Date: Sat Mar 20 09:50:01 2010 +0100
naoki: Remove old package.py module.
-----------------------------------------------------------------------
Summary of changes: naoki/__init__.py | 17 ++- naoki/backend.py | 50 +++++- naoki/chroot.py | 17 ++- naoki/constants.py | 20 ++- naoki/package.py | 449 ---------------------------------------------------- naoki/terminal.py | 3 +- pkgs/Constants | 3 +- pkgs/Functions | 2 + 8 files changed, 93 insertions(+), 468 deletions(-) delete mode 100644 naoki/package.py
Difference in files: diff --git a/naoki/__init__.py b/naoki/__init__.py index 40ca83c..9bafd46 100644 --- a/naoki/__init__.py +++ b/naoki/__init__.py @@ -72,7 +72,19 @@ class Naoki(object): else: package_names = args.packages
- packages = backend.parse_package(package_names, naoki=self) + packages = [] + for package in backend.parse_package(package_names, naoki=self): + if not force and package.built: + self.log.warn("Skipping %s which was already built" % package.name) + continue + + if not package.buildable: + self.log.error("%s is currently not buildable" % package.name) + self.log.error(" The package requires these packages to be built first: %s" \ + % [dep.name for dep in package.dependencies_unbuilt]) + continue + + packages.append(package)
if len(packages) >= 2: packages_sorted = backend.depsort(packages) @@ -122,7 +134,8 @@ Release : %(release)s Maintainer : %(maintainer)s License : %(license)s
-Files : %(objects)s +Files : %(files)s +Objects : %(objects)s Patches : %(patches)s --------------------------------------------------------------------------------\ """) diff --git a/naoki/backend.py b/naoki/backend.py index 0a12b5d..e4b2022 100644 --- a/naoki/backend.py +++ b/naoki/backend.py @@ -2,6 +2,7 @@
import os import urlgrabber +import urlgrabber.progress import urllib
import chroot @@ -40,7 +41,8 @@ def parse_package_info(names, toolchain=False): def parse_package(names, toolchain=False, naoki=None): packages = parse_package_info(names, toolchain)
- return [Package(package.name, naoki=naoki) for package in packages] + return [Package(package.name, naoki=naoki, toolchain=toolchain) \ + for package in packages]
def get_package_names(toolchain=False): if not __cache["package_names"]: @@ -167,6 +169,8 @@ class PackageInfo(object): self._name = name self.repo = repo
+ self.arch = arches.current["name"] + #def __cmp__(self, other): # return cmp(self.name, other.name)
@@ -187,7 +191,10 @@ class PackageInfo(object): def fetch(self): env = os.environ.copy() env.update(config.environment) - env["PKGROOT"] = PKGSDIR + env.update({ + "PKG_ARCH" : self.arch, + "PKGROOT" : PKGSDIR, + }) output = util.do("make -f %s" % self.filename, shell=True, cwd=os.path.join(PKGSDIR, self.repo.name, self.name), returnOutput=1, env=env)
@@ -214,6 +221,7 @@ class PackageInfo(object): "description" : self.description, "filename" : self.filename, "fingerprint" : self.fingerprint, + "files" : self.files, "group" : self.group, "license" : self.license, "maintainer" : self.maintainer, @@ -225,6 +233,18 @@ class PackageInfo(object): "version" : self.version, }
+ @property + def buildable(self): + return self.dependencies_unbuilt == [] + + @property + def built(self): + for file in self.package_files: + if not os.path.exists(os.path.join(PACKAGESDIR, file)): + return False + + return True + def _dependencies(self, s, recursive=False): c = s + "_CACHE" if not self._data.has_key(c): @@ -242,6 +262,24 @@ class PackageInfo(object): return self._dependencies("PKG_BUILD_DEPENDENCIES")
@property + def dependencies_built(self): + ret = [] + for dep in self.dependencies_all: + if dep.built: + ret.append(dep) + + return ret + + @property + def dependencies_unbuilt(self): + ret = [] + for dep in self.dependencies_all: + if not dep.built: + ret.append(dep) + + return ret + + @property def dependencies_all(self): return depsolve(self.dependencies + self.dependencies_build, recursive=True)
@@ -304,8 +342,8 @@ class PackageInfo(object):
class Package(object): - def __init__(self, name, naoki): - self.info = find_package_info(name) + def __init__(self, name, naoki, toolchain=False): + self.info = find_package_info(name, toolchain) self.naoki = naoki
#self.log.debug("Initialized package object %s" % name) @@ -327,7 +365,7 @@ class Package(object): download(self.info.objects, logger=self.log)
def extract(self, dest): - files = [os.path.join(PACKAGESDIR, file) for file in self.info.package_files] + files = [os.path.join(PACKAGESDIR, file) for file in self.package_files] if not files: return
@@ -342,7 +380,7 @@ class Package(object):
def get_repositories(toolchain=False): if toolchain: - return Repository("toolchain") + return [Repository("toolchain")]
repos = [] for repo in os.listdir(PKGSDIR): diff --git a/naoki/chroot.py b/naoki/chroot.py index caae03e..f63013d 100644 --- a/naoki/chroot.py +++ b/naoki/chroot.py @@ -6,7 +6,7 @@ import os import random import stat
-import package +import backend import util from constants import * from exception import * @@ -117,11 +117,16 @@ class Environment(object): % { "arch" : self.arch["name"], }, "BUILDROOT" : "/%s" % self.buildroot, "CHROOT" : "1", + "CFLAGS" : self.arch["cflags"], + "CXXFLAGS" : self.arch["cxxflags"], + "PKG_ARCH" : self.arch["name"], })
- if os.path.exists(self.chrootPath("usr", "ccache")): + ccache_path = os.path.join("tools_%s" % self.arch["name"], + "usr", "ccache", "bin") + if os.path.exists(self.chrootPath(ccache_path)): env.update({ - "PATH" : "/usr/ccache/bin:%s" % env["PATH"], + "PATH" : "/%s:%s" % (ccache_path, env["PATH"]), "CCACHE_DIR" : "/usr/src/ccache", })
@@ -329,9 +334,11 @@ class Toolchain(object): def build(self): self.log.info("Building toolchain...")
- packages = package.depsort(package.list(toolchain=True)) + packages = backend.get_package_names(toolchain=True) + packages = backend.parse_package(packages, toolchain=True) + packages = backend.depsort(packages) for pkg in packages: - if pkg.isBuilt: + if os.path.exists(os.path.join(self.path, pkg.name)): continue self.build_package(pkg) self.compress() diff --git a/naoki/constants.py b/naoki/constants.py index b36a471..a0dfac7 100644 --- a/naoki/constants.py +++ b/naoki/constants.py @@ -1,6 +1,7 @@ #!/usr/bin/python
import ConfigParser +import math import os
BASEDIR = os.getcwd() @@ -20,6 +21,19 @@ PATCHESDIR = os.path.join(CACHEDIR, "patches")
CONFIGFILE = os.path.join(CONFIGDIR, "naoki.conf")
+def calc_parallelism(): + """ + Calculate how many processes to run + at the same time. + + We take the log10(number of processors) * factor + """ + num = os.sysconf("SC_NPROCESSORS_CONF") + if num == 1: + return 2 + else: + return int(round(math.log10(num) * 26)) + class Config(object): _items = { "toolchain" : False, @@ -28,6 +42,7 @@ class Config(object): "core/glibc", ], "nice_level" : 0, + "parallelism" : calc_parallelism(), # # Cleanup settings "cleanup_on_failure" : False, @@ -78,7 +93,7 @@ class Config(object): def environment(self): return { "HOME" : os.environ.get("HOME", "/root"), - "TERM" : os.environ["TERM"], + "TERM" : os.environ.get("TERM", ""), "PS1" : os.environ.get("PS1", "\u:\w$ "), # "DISTRO_NAME" : self["distro_name"], @@ -87,8 +102,7 @@ class Config(object): "DISTRO_VERSION" : self["distro_version"], "DISTRO_SLOGAN" : self["distro_slogan"], # - "CFLAGS" : "-O2 -fomit-frame-pointer", - "PARALLELISMFLAGS" : "-j6", + "PARALLELISMFLAGS" : "-j%d" % self["parallelism"], }
diff --git a/naoki/package.py b/naoki/package.py deleted file mode 100644 index 5c250ea..0000000 --- a/naoki/package.py +++ /dev/null @@ -1,449 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import urlgrabber -import urlgrabber.progress - -import chroot -import terminal -import util - -from constants import * -from logger import getLog - -def list(toolchain=False): - pkgs = [] - for dir in os.listdir(PKGSDIR): - if not os.path.isdir(os.path.join(PKGSDIR, dir)): - continue - - # If we work in toolchain mode we don't return the other - # packages and if we are not, we don't return the toolchain packages. - if toolchain: - if dir != "toolchain": - continue - else: - if dir == "toolchain": - continue - - for package in os.listdir(os.path.join(PKGSDIR, dir)): - package = os.path.join(dir, package) - pkgs.append(Package(package)) - - pkgs.sort() - return pkgs - -def find(s, toolchain=False): - if not s: - return - - p = Package(s) - if p in list(toolchain): - return p - - for package in list(toolchain): - if os.path.basename(package.name) == s: - return package - -def groups(): - return [Group(name) for name in group_names()] - -def group_names(): - groups = [] - for package in list(): - group = package.group - if not group in groups: - groups.append(group) - groups.sort() - return groups - -def download(file, type): - if not file: - return - - dirs = { - "tarball" : TARBALLDIR, - } - filepath = os.path.join(dirs[type], file) - if os.path.exists(filepath): - return - - g = urlgrabber.grabber.URLGrabber( - user_agent = "%sSourceGrabber/%s" % (config["distro_name"], config["distro_version"],), - progress_obj = urlgrabber.progress.TextMeter(), - text = "Downloading %s..." % file, - ) - gobj = g.urlopen(config["download_%s_url" % type] % { "file" : file }) - - # XXX Need to check SHA1 sum here - - for dir in dirs.values(): - util.mkdir(dir) - - fobj = open(filepath, "w") - fobj.write(gobj.read()) - fobj.close() - gobj.close() - -def depsolve(packages, recursive=False): - deps = [] - for package in packages: - if not package: - continue - if package: - deps.append(package) - - if not recursive or not deps: - return deps - - while True: - length = len(deps) - for dep in deps[:]: - deps.extend(dep.deps) - - new_deps = [] - for dep in deps: - if not dep in new_deps: - new_deps.append(dep) - - deps = new_deps - - if length == len(deps): - break - - deps.sort() - return deps - -def deptree(packages): - ret = [packages] - - while True: - next = [] - stage = ret[-1][:] - for package in stage[:]: - for dep in package.getAllDeps(recursive=False): - if dep in ret[-1]: - stage.remove(package) - next.append(package) - break - - ret[-1] = stage - if next: - ret.append(next) - continue - - break - - return ret - -def depsort(packages): - ret = [] - for l1 in deptree(packages): - ret.extend(l1) - return ret - -class Package(object): - info_str = """\ -Name : %(name)s -Version : %(version)s -Release : %(release)s -Group : %(group)s - -%(summary)s - -Description : -%(description)s - -Maintainer : %(maintainer)s -License : %(license)s - -Built? : %(isBuilt)s -Can build? : %(canBuild)s - -Files : -%(objects)s - -Patches : -%(patches)s -""" - - info_wiki_str = """\ -====== %(name)s ====== -| **Version:** | %(version)s | -| **Release:** | %(release)s | -| **Group:** | %(group)s | -| **License:** | %(license)s | -| **Maintainer:** | %(maintainer)s | -| **Dependencies:** | %(deps)s | -| **Build dependencies:** | %(build_deps)s | -| %(summary)s || -| **Website:** | %(url)s | -""" - - def __init__(self, name): - self._name = name - - self.config = config - self.__fetch_data = None - - self._getAllDeps = None - - def __repr__(self): - return "<Package %s>" % self.name - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def fetch(self, key=None): - return self.__fetch()[key] - - def __fetch(self): - if not self.__fetch_data: - env = os.environ.copy() - env.update(config.environment) - env["PKGROOT"] = PKGSDIR - output = util.do("make -f %s" % self.filename, shell=True, - cwd=os.path.join(PKGSDIR, self.name), returnOutput=1, env=env) - - ret = {} - for line in output.splitlines(): - a = line.split("=", 1) - if not len(a) == 2: continue - key, val = a - ret[key] = val.strip(""") - - ret["FINGERPRINT"] = self.fingerprint - self.__fetch_data = ret - - return self.__fetch_data - - def download(self): - for object in self.objects: - download(object, type="tarball") - - @property - def fingerprint(self): - return str(os.stat(self.filename).st_mtime) - - @property - def filename(self): - return os.path.join(PKGSDIR, self.name, os.path.basename(self.name)) + ".nm" - - @property - def name(self): - return self._name - - @property - def version(self): - return self.fetch("PKG_VER") - - @property - def release(self): - return self.fetch("PKG_REL") - - @property - def summary(self): - return self.fetch("PKG_SUMMARY") - - @property - def description(self): - return self.fetch("PKG_DESCRIPTION") - - @property - def group(self): - return self.fetch("PKG_GROUP") - - @property - def packages(self): - return self.fetch("PKG_PACKAGES") - - @property - def package_files(self): - return sorted(self.fetch("PKG_PACKAGES_FILES").split(" ")) - - @property - def objects(self): - objects = [] - for object in sorted(self.fetch("PKG_OBJECTS").split(" ")): - if not object in self.patches: - objects.append(object) - return objects - - @property - def patches(self): - return sorted(self.fetch("PKG_PATCHES").split(" ")) - - @property - def maintainer(self): - return self.fetch("PKG_MAINTAINER") - - @property - def deps(self): - return self.getDeps() - - def getDeps(self, recursive=False): - deps = [] - for package in self.fetch("PKG_DEPENDENCIES").split(" "): - package = find(package) - if package: - deps.append(package) - return depsolve(deps, recursive) - - def getAllDeps(self, recursive=True): - if self.toolchain: - return depsolve(self.toolchain_deps, recursive) - - if not self._getAllDeps: - self._getAllDeps = depsolve(self.deps + self.build_deps, recursive) - - return self._getAllDeps - - @property - def build_deps(self): - deps = [] - for package in self.fetch("PKG_BUILD_DEPENDENCIES").split(" "): - package = find(package) - if package: - deps.append(package) - - deps.sort() - return deps - - @property - def toolchain_deps(self): - deps = [] - for package in self.fetch("PKG_TOOLCHAIN_DEPENDENCIES").split(" "): - package = find(package, toolchain=True) - if package: - deps.append(package) - - deps.sort() - return deps - - @property - def url(self): - return self.fetch("PKG_URL") - - @property - def id(self): - return "%s-%s-%s" % (self.name, self.version, self.release) - - @property - def license(self): - return self.fetch("PKG_LICENSE") - - @property - def __info(self): - return { - "name" : self.name, - "version" : self.version, - "release" : self.release, - "summary" : self.summary, - "description" : self.description, - "maintainer" : self.maintainer, - "objects" : self.objects, - "patches" : self.patches, - "group" : self.group, - "license" : self.license, - "isBuilt" : self.isBuilt, - "canBuild" : self.canBuild, - "url" : self.url, - } - - def info(self, long=False): - return self.info_str % self.__info - - def info_line(self, long=False): - if long: - s = "%-30s | %-15s | %s" % \ - (self.name, "%s-%s" % (self.version, self.release), self.summary) - - # Cut if text gets too long - columns = terminal.get_columns() - if len(s) >= columns: - s = s[:columns - 3] + "..." - - return s - - else: - return self.name - - def info_wiki(self, long=True): - if not long: - return " * [[.package:%(name)s|%(name)s]] - %(summary)s" % \ - { "name" : self.name, "summary" : self.summary, } - - __info = self.__info - __info.update({ - "deps" : "NOT IMPLEMENTED YET", - "build_deps" : "NOT IMPLEMENTED YET", - }) - - return self.info_wiki_str % __info - - @property - def isBuilt(self): - if self.toolchain: - return os.path.exists(self.toolchain_file) - - for item in self.package_files: - if not os.path.exists(os.path.join(PACKAGESDIR, item)): - return False - return True - - @property - def canBuild(self): - if self.toolchain: - for dep in self.toolchain_deps: - if not dep.isBuilt: - return False - return True - - deps = self.deps + self.build_deps - for dep in deps: - if not dep.isBuilt: - return False - - return True - - def extract(self, dest): - files = [os.path.join(PACKAGESDIR, file) for file in self.package_files] - if not files: - return - - getLog().debug("Extracting %s..." % self.name) - util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"), - dest, " ".join(files)), shell=True) - - @property - def toolchain(self): - if self.name.startswith("toolchain"): - return True - return False - - @property - def toolchain_file(self): - return os.path.join(TOOLCHAINSDIR, "tools_i686", "built", self.id) - - -class Group(object): - def __init__(self, name): - self.name = name - - self.__packages = [] - - def wiki_headline(self): - return "===== %s =====" % self.name - - @property - def packages(self): - if not self.__packages: - for pkg in list(): - if not pkg.group == self.name: - continue - self.__packages.append(pkg) - - self.__packages.sort() - - return self.__packages diff --git a/naoki/terminal.py b/naoki/terminal.py index be7ddc5..5ed1048 100644 --- a/naoki/terminal.py +++ b/naoki/terminal.py @@ -176,7 +176,8 @@ class Commandline(object): # Set debugging mode self.naoki.logging.debug(args.debug)
- # XXX Set architecture + # Set architecture + arches.set(args.arch)
def __parse(self): parser = Parser("root", diff --git a/pkgs/Constants b/pkgs/Constants index 47a9d63..93798f1 100644 --- a/pkgs/Constants +++ b/pkgs/Constants @@ -33,8 +33,7 @@ PKG_NAME_REAL = $(notdir $(CURDIR)) PKG_PACKAGES = $(PKG_NAME_REAL) PKG_PACKAGES_FILES = $(foreach package,$(PKG_PACKAGES),$(call DO_PACKAGE_FILENAME,$(package)))
-DO_PACKAGE_FILENAME = $(1)$(PKG_SUFFIX) -PKG_SUFFIX = -$(PKG_VER)-$(DISTRO_SNAME)$(DISTRO_EPOCH)-$(PKG_REL).ipk +PKG_SUFFIX = -$(PKG_VER)-$(DISTRO_SNAME)$(DISTRO_EPOCH)-$(PKG_ARCH).$(PKG_REL).ipk
DO_EXTRACT = $(DIR_TOOLS)/extractor DO_PATCHES = cd $(DIR_APP) && $(DIR_TOOLS)/patch $(foreach patch,$(PKG_PATCHES),$(DIR_PATCHES)/$(patch)) diff --git a/pkgs/Functions b/pkgs/Functions index 867de5a..e79a5a0 100644 --- a/pkgs/Functions +++ b/pkgs/Functions @@ -7,6 +7,8 @@
include $(PKGROOT)/gmsl
+DO_PACKAGE_FILENAME = $(1)$(PKG_SUFFIX) + define DO_FILELIST @echo "# Filelist dump" @cd $(BUILDROOT) && find -ls
hooks/post-receive -- IPFire 3.x development tree