Module: Python::Pickle

Defined in:
lib/python/pickle.rb,
lib/python/pickle/tuple.rb,
lib/python/pickle/version.rb,
lib/python/pickle/protocol.rb,
lib/python/pickle/py_class.rb,
lib/python/pickle/protocol0.rb,
lib/python/pickle/protocol1.rb,
lib/python/pickle/protocol2.rb,
lib/python/pickle/protocol3.rb,
lib/python/pickle/protocol4.rb,
lib/python/pickle/protocol5.rb,
lib/python/pickle/py_object.rb,
lib/python/pickle/byte_array.rb,
lib/python/pickle/exceptions.rb,
lib/python/pickle/instruction.rb,
lib/python/pickle/deserializer.rb,
lib/python/pickle/instructions/dup.rb,
lib/python/pickle/instructions/get.rb,
lib/python/pickle/instructions/int.rb,
lib/python/pickle/instructions/pop.rb,
lib/python/pickle/instructions/put.rb,
lib/python/pickle/instructions/dict.rb,
lib/python/pickle/instructions/ext1.rb,
lib/python/pickle/instructions/ext2.rb,
lib/python/pickle/instructions/ext4.rb,
lib/python/pickle/instructions/list.rb,
lib/python/pickle/instructions/long.rb,
lib/python/pickle/instructions/mark.rb,
lib/python/pickle/instructions/none.rb,
lib/python/pickle/instructions/stop.rb,
lib/python/pickle/instructions/build.rb,
lib/python/pickle/instructions/float.rb,
lib/python/pickle/instructions/frame.rb,
lib/python/pickle/instructions/long1.rb,
lib/python/pickle/instructions/long4.rb,
lib/python/pickle/instructions/proto.rb,
lib/python/pickle/instructions/tuple.rb,
lib/python/pickle/instructions/append.rb,
lib/python/pickle/instructions/global.rb,
lib/python/pickle/instructions/reduce.rb,
lib/python/pickle/instructions/string.rb,
lib/python/pickle/instructions/tuple1.rb,
lib/python/pickle/instructions/tuple2.rb,
lib/python/pickle/instructions/tuple3.rb,
lib/python/pickle/instructions/appends.rb,
lib/python/pickle/instructions/bin_get.rb,
lib/python/pickle/instructions/bin_put.rb,
lib/python/pickle/instructions/memoize.rb,
lib/python/pickle/instructions/new_obj.rb,
lib/python/pickle/instructions/bin_int1.rb,
lib/python/pickle/instructions/new_true.rb,
lib/python/pickle/instructions/pop_mark.rb,
lib/python/pickle/instructions/set_item.rb,
lib/python/pickle/instructions/add_items.rb,
lib/python/pickle/instructions/bin_bytes.rb,
lib/python/pickle/instructions/bin_float.rb,
lib/python/pickle/instructions/empty_set.rb,
lib/python/pickle/instructions/has_value.rb,
lib/python/pickle/instructions/new_false.rb,
lib/python/pickle/instructions/set_items.rb,
lib/python/pickle/instructions/bin_bytes8.rb,
lib/python/pickle/instructions/bin_string.rb,
lib/python/pickle/instructions/empty_dict.rb,
lib/python/pickle/instructions/empty_list.rb,
lib/python/pickle/instructions/frozen_set.rb,
lib/python/pickle/instructions/new_obj_ex.rb,
lib/python/pickle/instructions/bin_unicode.rb,
lib/python/pickle/instructions/byte_array8.rb,
lib/python/pickle/instructions/empty_tuple.rb,
lib/python/pickle/instructions/next_buffer.rb,
lib/python/pickle/instructions/bin_unicode8.rb,
lib/python/pickle/instructions/long_bin_get.rb,
lib/python/pickle/instructions/stack_global.rb,
lib/python/pickle/instructions/readonly_buffer.rb,
lib/python/pickle/instructions/short_bin_bytes.rb,
lib/python/pickle/instructions/short_bin_string.rb,
lib/python/pickle/instructions/short_bin_unicode.rb,
lib/python/pickle/instructions/has_length_and_value.rb

Overview

A modern Ruby implementation of the Python Pickle serialization format.

Defined Under Namespace

Modules: Instructions Classes: ByteArray, DeserializationError, Deserializer, Error, Instruction, InvalidFormat, Protocol, Protocol0, Protocol1, Protocol2, Protocol3, Protocol4, Protocol5, PyClass, PyObject, Tuple

Constant Summary collapse

PROTOCOL_VERSIONS =

This constant is part of a private API. You should avoid using this constant if possible, as it may be removed or be changed in the future.

Mapping of protocol versions to protocol parsers.

{
  0 => Protocol0,
  1 => Protocol1,
  2 => Protocol2,
  3 => Protocol3,
  4 => Protocol4,
  5 => Protocol5
}
DEFAULT_PROTOCOL =

The default protocol version to use.

4
HIGHEST_PROTOCOL =

The highest protocol version supported.

5
VERSION =

python-pickle version

'0.1.1'

Class Method Summary collapse

Class Method Details

.dump(object, output = nil, protocol: DEFAULT_PROTOCOL) ⇒ Object

Note:

serializing is currently not supported.

Serializes the Ruby object into Python Pickle data.

Parameters:

  • object (Object) —

    The Ruby object to serialize.

  • output (IO) (defaults to: nil) —

    The option output to write the Pickle data to.

  • protocol (Integer) (defaults to: DEFAULT_PROTOCOL) —

    The desired Python Pickle protocol to use.

Raises:

  • (NotImplementedError) 


162
163
164
 
# File 'lib/python/pickle.rb', line 162

def self.dump(object,output=nil, protocol: DEFAULT_PROTOCOL)
  raise(NotImplementedError,"pickle serializing is currently not supported")
end

.infer_protocol_version(io) ⇒ Integer

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Infers the protocol version from the IO stream.

Parameters:

  • io (IO) —

    The IO stream to inspect.

Returns:

  • (Integer) —

    The inferred Python Pickle protocol version.

Raises:

  • (InvalidFormat) —

    Could not determine the Pickle version from the first two bytes of the IO stream.



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/python/pickle.rb', line 181

def self.infer_protocol_version(io)
  opcode = io.getbyte

  begin
    case opcode
    when 0x80 # PROTO (added in protocol 2)
      version = io.getbyte
      io.ungetbyte(version)
      return version
    when 48,  # POP (protocol 0)
         50,  # DUP (protocol 0)
         70,  # FLOAT (protocol 0)
         83,  # STRING (protocol 0)
         86,  # UNICODE (protocol 0)
         100, # DICT (protocol 0)
         103, # GET (protocol 0)
         108, # LIST (protocol 0)
         112  # PUT (protocol 0)
      0
    when 41,  # EMPTY_TUPLE (protocol 1)
         71,  # BINFLOAT (protocol 1)
         75,  # BININT1 (protocol 1)
         84,  # BINSTRING (protocol 1)
         85,  # SHORT_BINSTRING (protocol 1)
         88,  # BINUNICODE (protocol 1)
         93,  # EMPTY_LIST (protocol 1)
         101, # APPENDS (protocol 1)
         113, # BINPUT (protocol 1)
         117, # SETITEMS (protocol 1)
         125  # EMPTY_DICT (protocol 1)
      1
    when 46 # STOP
      # if we've read all the way to the end of the stream and still cannot
      # find any protocol 0 or protocol 1 specific opcodes, assume protocol 0
      0
    when 73, # INT  (identical in both protocol 0 and 1)
         76  # LONG (identical in both protocol 0 and 1)
      chars = io.gets

      begin
        infer_protocol_version(io)
      ensure
        chars.each_byte.reverse_each { |b| io.ungetbyte(b) }
      end
    when 40,  # MARK    (identical in both protocol 0 and 1)
         78,  # NONE    (identical in both protocol 0 and 1)
         82,  # REDUCE  (identical in both protocol 0 and 1)
         97,  # APPEND  (identical in both protocol 0 and 1)
         98,  # BUILD   (identical in both protocol 0 and 1)
         115, # SETITEM (identical in both protocol 0 and 1)
         116  # TUPLE   (identical in both protocol 0 and 1)
      infer_protocol_version(io)
    when 99 # GLOBAL
      first_nl_string  = io.gets
      second_nl_string = io.gets

      begin
        infer_protocol_version(io)
      ensure
        # push the read bytes back into the IO stream
        second_nl_string.each_byte.reverse_each { |b| io.ungetbyte(b) }
        first_nl_string.each_byte.reverse_each  { |b| io.ungetbyte(b) }
      end
    else
      raise(InvalidFormat,"cannot infer protocol version from opcode (#{opcode.inspect}) at position #{io.pos}")
    end
  ensure
    io.ungetbyte(opcode)
  end
end

.load(data, protocol: nil, **kwargs) ⇒ Object

Deserializes the Python Pickle stream into a Ruby object.

Parameters:

  • data (String, IO) —

    The Python pickle stream to parse.

  • protocol (Integer, nil) (defaults to: nil) —

    The explicit protocol version to use. If nil the protocol version will be inferred by inspecting the first two bytes of the stream.

  • kwargs (Hash{Symbol => Object}) —

    Additional keyword arguments.

Options Hash (**kwargs):

  • :extensions (Hash{Integer => Object}) —

    A Hash of registered extension IDs and their Objects.

  • :constants (Hash{String => Hash{String => Class,Method}}) —

    An optional mapping of custom Python constant names to Ruby classes or methods.

Raises:



103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/python/pickle.rb', line 103

def self.load(data, protocol: nil, **kwargs)
  deserializer = Deserializer.new(**kwargs)

  parse(data, protocol: protocol) do |instruction|
    status, object = deserializer.execute(instruction)

    if status == :halt
      return object
    end
  end

  raise(DeserializationError,"failed to deserialize any object data from stream")
end

.load_file(path, **kwargs) ⇒ Object

Deserializes a Python Pickle file.

Parameters:

  • path (String) —

    The path of the file.

  • kwargs (Hash{Symbol => Object}) —

    Additional keyword arguments.

Options Hash (**kwargs):

  • :extensions (Hash{Integer => Object}) —

    A Hash of registered extension IDs and their Objects.

  • :constants (Hash{String => Hash{String => Class,Method}}) —

    An optional mapping of custom Python constant names to Ruby classes or methods.

Returns:

  • (Object) —

    The deserialized object.



136
137
138
139
140
141
142
143
144
 
# File 'lib/python/pickle.rb', line 136

def self.load_file(path,**kwargs)
  result = nil

  File.open(path,'rb') do |file|
    result = load(file,**kwargs)
  end

  return result
end

.parse(data, protocol: nil) {|instruction| ... } ⇒ Array<Instruction>

Parses a Python pickle stream.

Parameters:

  • data (String, IO) —

    The Python pickle stream to parse.

  • protocol (Integer, nil) (defaults to: nil) —

    The explicit protocol version to use. If nil the protocol version will be inferred by inspecting the first two bytes of the stream.

Yields:

  • (instruction) —

    If a block is given, it will be passed each parsed Pickle instruction.

Yield Parameters:

  • instruction (Instruction) —

    A parsed Pickle instruction from the Pickle stream.

Returns:

  • (Array<Instruction>) —

    All parsed Pickle instructions from the Pickle stream.



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/python/pickle.rb', line 59

def self.parse(data, protocol: nil, &block)
  io = case data 
       when String then StringIO.new(data)
       when IO     then data
       else
         raise(ArgumentError,"argument must be either an IO object or a String: #{io.inspect}")
       end

  if protocol
    if (protocol < 0) || (protocol > HIGHEST_PROTOCOL)
      raise(ArgumentError,"protocol must be between 0 or #{HIGHEST_PROTOCOL}, but was #{protocol.inspect}")
    end
  else
    protocol = infer_protocol_version(io)
  end

  protocol_class = PROTOCOL_VERSIONS.fetch(protocol)
  protocol       = protocol_class.new(io)

  return protocol.read(&block)
end