Class: FPM::Package::Python

Inherits:
FPM::Package show all
Defined in:
lib/fpm/package/python.rb

Overview

Support for python packages.

This supports input, but not output.

Example:

# Download the django python package:
pkg = FPM::Package::Python.new
pkg.input("Django")

Defined Under Namespace

Classes: PythonMetadata

Instance Attribute Summary

Attributes inherited from FPM::Package

#architecture, #attributes, #attrs, #category, #config_files, #conflicts, #dependencies, #description, #directories, #epoch, #iteration, #license, #maintainer, #name, #provides, #replaces, #scripts, #url, #vendor, #version

Instance Method Summary collapse

Methods inherited from FPM::Package

apply_options, #build_path, #cleanup, #cleanup_build, #cleanup_staging, #convert, #converted_from, default_attributes, #edit_file, #files, inherited, #initialize, option, #output, #script, #staging_path, #to_s, #type, type, types

Methods included from Util

#ar_cmd, #ar_cmd_deterministic?, #copied_entries, #copy_entry, #copy_metadata, #default_shell, #erbnew, #execmd, #expand_pessimistic_constraints, #logger, #program_exists?, #program_in_path?, #safesystem, #safesystemout, #tar_cmd, #tar_cmd_supports_sort_names_and_set_mtime?

Constructor Details

This class inherits a constructor from FPM::Package

Instance Method Details

#download_if_necessary(package, version = nil) ⇒ Object

Download the given package if necessary. If version is given, that version will be downloaded, otherwise the latest is fetched.



395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
 
# File 'lib/fpm/package/python.rb', line 395

def download_if_necessary(package, version=nil)
  path = package

  # If it's a path, assume local build.
  if File.exist?(path)
    return path if File.directory?(path)

    basename = File.basename(path)
    return File.dirname(path) if basename == "pyproject.toml"
    return File.dirname(path) if basename == "setup.py"

    return path if path.end_with?(".tar.gz")
    return path if path.end_with?(".tgz") # amqplib v1.0.2 does this
    return path if path.end_with?(".whl")
    return path if path.end_with?(".zip")
    return path if File.exist?(File.join(path, "setup.py"))
    return path if File.exist?(File.join(path, "pyproject.toml"))

    raise [
      "Local file doesn't appear to be a supported type for a python package. Expected one of:",
      "  - A directory containing setup.py or pyproject.toml",
      "  - A file ending in .tar.gz (a python source dist)",
      "  - A file ending in .whl (a python wheel)",
    ].join("\n")
  end

  logger.info("Trying to download", :package => package)

  if version.nil?
    want_pkg = "#{package}"
  else
    want_pkg = "#{package}==#{version}"
  end

  target = build_path(package)
  FileUtils.mkdir(target) unless File.directory?(target)

  # attributes[:python_pip] -- expected to be a path
  if attributes[:python_pip]
    logger.debug("using pip", :pip => attributes[:python_pip])
    pip = [attributes[:python_pip]] if pip.is_a?(String)
    setup_cmd = [
      *attributes[:python_pip],
      "download",
      "--no-clean",
      "--no-deps",
      "-d", target,
      "-i", attributes[:python_pypi],
    ]

    if attributes[:python_trusted_host]
      setup_cmd += [
        "--trusted-host",
        attributes[:python_trusted_host],
      ]
    end

    setup_cmd << want_pkg

    safesystem(*setup_cmd)

    files = ::Dir.entries(target).filter { |entry| entry =~ /\.(whl|tgz|tar\.gz|zip)$/ }
    if files.length != 1
      raise "Unexpected directory layout after `pip download ...`. This might be an fpm bug? The directory contains these files: #{files.inspect}"
    end
    return File.join(target, files.first)
  else
    # no pip, use easy_install
    logger.debug("no pip, defaulting to easy_install", :easy_install => attributes[:python_easyinstall])
    safesystem(attributes[:python_easyinstall], "-i",
               attributes[:python_pypi], "--editable", "-U",
               "--build-directory", target, want_pkg)
    # easy_install will put stuff in @tmpdir/packagename/, so find that:
    #  @tmpdir/somepackage/setup.py
    #dirs = ::Dir.glob(File.join(target, "*"))
    files = ::Dir.entries(target).filter { |entry| entry != "." && entry != ".." }
    if dirs.length != 1
      raise "Unexpected directory layout after easy_install. Maybe file a bug? The directory is #{build_path}"
    end
    return dirs.first
  end
end

#explore_environment ⇒ Object

def input



366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/fpm/package/python.rb', line 366

def explore_environment
  if !attributes[:python_bin_given?]
    # If --python-bin isn't set, try to find a good default python executable path, because it might not be "python"
    pythons = [ "python", "python3", "python2" ]
    default_python = pythons.find { |py| program_exists?(py) }

    if default_python.nil?
      raise FPM::Util::ExecutableNotFound, "Could not find any python interpreter. Tried the following: #{pythons.join(", ")}"
    end

    logger.info("Setting default python executable", :name => default_python)
    attributes[:python_bin] = default_python

    if !attributes[:python_package_name_prefix_given?]
      attributes[:python_package_name_prefix] = default_python
      logger.info("Setting package name prefix", :name => default_python)
    end
  end

  if attributes[:python_internal_pip?]
    # XXX: Should we detect if internal pip is available?
    attributes[:python_pip] = [ attributes[:python_bin], "-m", "pip"]
  end
end

#fix_name(name) ⇒ Object

Sanitize package name. Some PyPI packages can be named ‘python-foo’, so we don’t want to end up with a package named ‘python-python-foo’. But we want packages named like ‘pythonweb’ to be suffixed ‘python-pythonweb’.



587
588
589
590
591
592
593
594
595
 
# File 'lib/fpm/package/python.rb', line 587

def fix_name(name)
  if name.start_with?("python")
    # If the python package is called "python-foo" strip the "python-" part while
    # prepending the package name prefix.
    return [attributes[:python_package_name_prefix], name.gsub(/^python-/, "")].join("-")
  else
    return [attributes[:python_package_name_prefix], name].join("-")
  end
end

#input(package) ⇒ Object

Input a package.

The ‘package’ can be any of:

  • A name of a package on pypi (ie; easy_install some-package)

  • The path to a directory containing setup.py or pyproject.toml

  • The path to a setup.py or pyproject.toml

  • The path to a python sdist file ending in .tar.gz

  • The path to a python wheel file ending in .whl



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
 
# File 'lib/fpm/package/python.rb', line 319

def input(package)
  explore_environment

  path_to_package = download_if_necessary(package, version)

  # Expect a setup.py or pyproject.toml if it's a directory.
  if File.directory?(path_to_package)
    if !(File.exist?(File.join(path_to_package, "setup.py")) or File.exist?(File.join(path_to_package, "pyproject.toml")))
      raise FPM::InvalidPackageConfiguration, "The path ('#{path_to_package}') doesn't appear to be a python package directory. I expected either a pyproject.toml or setup.py but found neither."
    end
  end

  if File.file?(path_to_package)
    if ["setup.py", "pyproject.toml"].include?(File.basename(path_to_package))
      path_to_package = File.dirname(path_to_package)
    end
  end

  if [".tar.gz", ".tgz"].any? { |suffix| path_to_package.end_with?(suffix) }
    # Have pip convert the .tar.gz (source dist?) into a wheel
    logger.debug("Found tarball and assuming it's a python source package.")
    safesystem(*attributes[:python_pip], "wheel", "--no-deps", "-w", build_path, path_to_package)

    path_to_package = ::Dir.glob(build_path("*.whl")).first
    if path_to_package.nil?
      raise FPM::InvalidPackageConfiguration, "Failed building python package format - fpm tried to build a python wheel, but didn't find the .whl file. This might be a bug in fpm."
    end
  elsif File.directory?(path_to_package)
    logger.debug("Found directory and assuming it's a python source package.")
    safesystem(*attributes[:python_pip], "wheel", "--no-deps", "-w", build_path, path_to_package)

    if attributes[:python_obey_requirements_txt?]
      reqtxt = File.join(path_to_package, "requirements.txt")
      @requirements_txt = File.read(reqtxt).split("\n") if File.file?(reqtxt)
    end

    path_to_package = ::Dir.glob(build_path("*.whl")).first
    if path_to_package.nil?
      raise FPM::InvalidPackageConfiguration, "Failed building python package format - fpm tried to build a python wheel, but didn't find the .whl file. This might be a bug in fpm."
    end

  end

  load_package_info(path_to_package)
  install_to_staging(path_to_package)
end

#install_to_staging(path) ⇒ Object

Install this package to the staging directory



598
599
600
601
602
603
604
605
606
607
608
609
 
# File 'lib/fpm/package/python.rb', line 598

def install_to_staging(path)
  prefix = "/"
  prefix = attributes[:prefix] unless attributes[:prefix].nil?

  # XXX: Note: pip doesn't seem to have any equivalent to `--install-lib` or similar flags.
  # XXX: Deprecate :python_install_data, :python_install_lib, :python_install_bin
  # XXX: Deprecate: :python_setup_py_arguments
  flags = [ "--root", staging_path ]
  flags += [ "--prefix", prefix ] if !attributes[:prefix].nil?

  safesystem(*attributes[:python_pip], "install", "--no-deps", *flags, path)
end

#load_package_info(path) ⇒ Object

Load the package information like name, version, dependencies.



479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
 
# File 'lib/fpm/package/python.rb', line 479

def load_package_info(path)
  if path.end_with?(".whl")
    # XXX: Maybe use rubyzip to parse the .whl (zip) file instead?
    metadata = nil
    execmd(["unzip", "-p", path, "*.dist-info/METADATA"], :stdin => false, :stderr => false) do |stdout|
      metadata = PythonMetadata.from(stdout.read(64<<10))
    end

    wheeldata = nil
    execmd(["unzip", "-p", path, "*.dist-info/WHEEL"], :stdin => false, :stderr => false) do |stdout|
      wheeldata, _ = PythonMetadata.parse(stdout.read(64<<10))
    end
  else
    raise "Unexpected python package path. This might be an fpm bug? The path is #{path}"
  end

  self.architecture = wheeldata["Root-Is-Purelib"] == "true" ? "all" : "native"

  self.description = metadata.description unless metadata.description.nil?
  self.license = metadata.license unless metadata.license.nil?
  self.version = metadata.version
  self.url = metadata.homepage unless metadata.homepage.nil?

  self.name = metadata.name

  # name prefixing is optional, if enabled, a name 'foo' will become
  # 'python-foo' (depending on what the python_package_name_prefix is)
  self.name = fix_name(self.name) if attributes[:python_fix_name?] 

  # convert python-Foo to python-foo if flag is set
  self.name = self.name.downcase if attributes[:python_downcase_name?]

  self.maintainer = metadata.maintainer 

  if !attributes[:no_auto_depends?] and attributes[:python_dependencies?]
    # Python Dependency specifiers are a somewhat complex format described here:
    # https://packaging.python.org/en/latest/specifications/dependency-specifiers/#environment-markers
    #
    # We can ask python's packaging module to parse and evaluate these.
    # XXX: Allow users to override environnment values.
    #
    # Example:
    # Requires-Dist: tzdata; sys_platform = win32
    # Requires-Dist: asgiref>=3.8.1

    dep_re = /^([^<>!= ]+)\s*(?:([~<>!=]{1,2})\s*(.*))?$/

    reqs = []

    # --python-obey-requirements-txt should use requirements.txt
    # (if found in the python package) and  replace the requirments listed from the metadata
    if attributes[:python_obey_requirements_txt?] && !@requirements_txt.nil?
      requires = @requirements_txt
    else
      requires = metadata.requires
    end

    # Evaluate python package requirements and only show ones matching the current environment
    # (Environment markers, etc)
    # Additionally, 'extra' features such as a requirement named `django[bcrypt]` isn't quite supported yet,
    # since the marker.evaluate() needs to be passed some environment like { "extra": "bcrypt" }
    execmd([attributes[:python_bin], File.expand_path(File.join("pyfpm", "parse_requires.py"), File.dirname(__FILE__))]) do |stdin, stdout, stderr|
      requires.each { |r| stdin.puts(r) }
      stdin.close
      data = stdout.read
      logger.pipe(stderr => :warn)
      reqs += JSON.parse(data)
    end

    reqs.each do |dep|
      match = dep_re.match(dep)
      if match.nil?
        logger.error("Unable to parse dependency", :dependency => dep)
        raise FPM::InvalidPackageConfiguration, "Invalid dependency '#{dep}'"
      end

      name, cmp, version = match.captures

      next if attributes[:python_disable_dependency].include?(name)

      # convert == to =
      if cmp == "==" or cmp == "~="
        logger.info("Converting == dependency requirement to =", :dependency => dep )
        cmp = "="
      end

      # dependency name prefixing is optional, if enabled, a name 'foo' will
      # become 'python-foo' (depending on what the python_package_name_prefix
      # is)
      name = fix_name(name) if attributes[:python_fix_dependencies?]

      # convert dependencies from python-Foo to python-foo
      name = name.downcase if attributes[:python_downcase_dependencies?]

      if cmp.nil? && version.nil?
        self.dependencies << "#{name}"
      else
        self.dependencies << "#{name} #{cmp} #{version}"
      end
    end # parse Requires-Dist dependencies
  end # if attributes[:python_dependencies?]
end