1573 lines
165 KiB
HTML
1573 lines
165 KiB
HTML
<!DOCTYPE html>
|
||
|
||
<html lang="en" data-content_root="../">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta property="og:title" content="pickle — Python object serialization" />
|
||
<meta property="og:type" content="website" />
|
||
<meta property="og:url" content="https://docs.python.org/3/library/pickle.html" />
|
||
<meta property="og:site_name" content="Python documentation" />
|
||
<meta property="og:description" content="Source code: Lib/pickle.py The pickle module implements binary protocols for serializing and de-serializing a Python object structure. “Pickling” is the process whereby a Python object hierarchy is..." />
|
||
<meta property="og:image" content="https://docs.python.org/3/_static/og-image.png" />
|
||
<meta property="og:image:alt" content="Python documentation" />
|
||
<meta name="description" content="Source code: Lib/pickle.py The pickle module implements binary protocols for serializing and de-serializing a Python object structure. “Pickling” is the process whereby a Python object hierarchy is..." />
|
||
<meta property="og:image:width" content="200">
|
||
<meta property="og:image:height" content="200">
|
||
<meta name="theme-color" content="#3776ab">
|
||
|
||
<title>pickle — Python object serialization — Python 3.13.3 documentation</title><meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/pydoctheme.css?v=23252803" />
|
||
<link id="pygments_dark_css" media="(prefers-color-scheme: dark)" rel="stylesheet" type="text/css" href="../_static/pygments_dark.css?v=5349f25f" />
|
||
|
||
<script src="../_static/documentation_options.js?v=5d57ca2d"></script>
|
||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
|
||
<script src="../_static/sidebar.js"></script>
|
||
|
||
<link rel="search" type="application/opensearchdescription+xml"
|
||
title="Search within Python 3.13.3 documentation"
|
||
href="../_static/opensearch.xml"/>
|
||
<link rel="author" title="About these documents" href="../about.html" />
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
<link rel="copyright" title="Copyright" href="../copyright.html" />
|
||
<link rel="next" title="copyreg — Register pickle support functions" href="copyreg.html" />
|
||
<link rel="prev" title="Data Persistence" href="persistence.html" />
|
||
|
||
<link rel="canonical" href="https://docs.python.org/3/library/pickle.html">
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@media only screen {
|
||
table.full-width-table {
|
||
width: 100%;
|
||
}
|
||
}
|
||
</style>
|
||
<link rel="stylesheet" href="../_static/pydoctheme_dark.css" media="(prefers-color-scheme: dark)" id="pydoctheme_dark_css">
|
||
<link rel="shortcut icon" type="image/png" href="../_static/py.svg" />
|
||
<script type="text/javascript" src="../_static/copybutton.js"></script>
|
||
<script type="text/javascript" src="../_static/menu.js"></script>
|
||
<script type="text/javascript" src="../_static/search-focus.js"></script>
|
||
<script type="text/javascript" src="../_static/themetoggle.js"></script>
|
||
<script type="text/javascript" src="../_static/rtd_switcher.js"></script>
|
||
<meta name="readthedocs-addons-api-version" content="1">
|
||
|
||
</head>
|
||
<body>
|
||
<div class="mobile-nav">
|
||
<input type="checkbox" id="menuToggler" class="toggler__input" aria-controls="navigation"
|
||
aria-pressed="false" aria-expanded="false" role="button" aria-label="Menu" />
|
||
<nav class="nav-content" role="navigation">
|
||
<label for="menuToggler" class="toggler__label">
|
||
<span></span>
|
||
</label>
|
||
<span class="nav-items-wrapper">
|
||
<a href="https://www.python.org/" class="nav-logo">
|
||
<img src="../_static/py.svg" alt="Python logo"/>
|
||
</a>
|
||
<span class="version_switcher_placeholder"></span>
|
||
<form role="search" class="search" action="../search.html" method="get">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" class="search-icon">
|
||
<path fill-rule="nonzero" fill="currentColor" d="M15.5 14h-.79l-.28-.27a6.5 6.5 0 001.48-5.34c-.47-2.78-2.79-5-5.59-5.34a6.505 6.505 0 00-7.27 7.27c.34 2.8 2.56 5.12 5.34 5.59a6.5 6.5 0 005.34-1.48l.27.28v.79l4.25 4.25c.41.41 1.08.41 1.49 0 .41-.41.41-1.08 0-1.49L15.5 14zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z"></path>
|
||
</svg>
|
||
<input placeholder="Quick search" aria-label="Quick search" type="search" name="q" />
|
||
<input type="submit" value="Go"/>
|
||
</form>
|
||
</span>
|
||
</nav>
|
||
<div class="menu-wrapper">
|
||
<nav class="menu" role="navigation" aria-label="main navigation">
|
||
<div class="language_switcher_placeholder"></div>
|
||
|
||
<label class="theme-selector-label">
|
||
Theme
|
||
<select class="theme-selector" oninput="activateTheme(this.value)">
|
||
<option value="auto" selected>Auto</option>
|
||
<option value="light">Light</option>
|
||
<option value="dark">Dark</option>
|
||
</select>
|
||
</label>
|
||
<div>
|
||
<h3><a href="../contents.html">Table of Contents</a></h3>
|
||
<ul>
|
||
<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> — Python object serialization</a><ul>
|
||
<li><a class="reference internal" href="#relationship-to-other-python-modules">Relationship to other Python modules</a><ul>
|
||
<li><a class="reference internal" href="#comparison-with-marshal">Comparison with <code class="docutils literal notranslate"><span class="pre">marshal</span></code></a></li>
|
||
<li><a class="reference internal" href="#comparison-with-json">Comparison with <code class="docutils literal notranslate"><span class="pre">json</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#data-stream-format">Data stream format</a></li>
|
||
<li><a class="reference internal" href="#module-interface">Module Interface</a></li>
|
||
<li><a class="reference internal" href="#what-can-be-pickled-and-unpickled">What can be pickled and unpickled?</a></li>
|
||
<li><a class="reference internal" href="#pickling-class-instances">Pickling Class Instances</a><ul>
|
||
<li><a class="reference internal" href="#persistence-of-external-objects">Persistence of External Objects</a></li>
|
||
<li><a class="reference internal" href="#dispatch-tables">Dispatch Tables</a></li>
|
||
<li><a class="reference internal" href="#handling-stateful-objects">Handling Stateful Objects</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#custom-reduction-for-types-functions-and-other-objects">Custom Reduction for Types, Functions, and Other Objects</a></li>
|
||
<li><a class="reference internal" href="#out-of-band-buffers">Out-of-band Buffers</a><ul>
|
||
<li><a class="reference internal" href="#provider-api">Provider API</a></li>
|
||
<li><a class="reference internal" href="#consumer-api">Consumer API</a></li>
|
||
<li><a class="reference internal" href="#example">Example</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#restricting-globals">Restricting Globals</a></li>
|
||
<li><a class="reference internal" href="#performance">Performance</a></li>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
<div>
|
||
<h4>Previous topic</h4>
|
||
<p class="topless"><a href="persistence.html"
|
||
title="previous chapter">Data Persistence</a></p>
|
||
</div>
|
||
<div>
|
||
<h4>Next topic</h4>
|
||
<p class="topless"><a href="copyreg.html"
|
||
title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code> — Register <code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> support functions</a></p>
|
||
</div>
|
||
<div role="note" aria-label="source link">
|
||
<h3>This Page</h3>
|
||
<ul class="this-page-menu">
|
||
<li><a href="../bugs.html">Report a Bug</a></li>
|
||
<li>
|
||
<a href="https://github.com/python/cpython/blob/main/Doc/library/pickle.rst"
|
||
rel="nofollow">Show Source
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div class="related" role="navigation" aria-label="Related">
|
||
<h3>Navigation</h3>
|
||
<ul>
|
||
<li class="right" style="margin-right: 10px">
|
||
<a href="../genindex.html" title="General Index"
|
||
accesskey="I">index</a></li>
|
||
<li class="right" >
|
||
<a href="../py-modindex.html" title="Python Module Index"
|
||
>modules</a> |</li>
|
||
<li class="right" >
|
||
<a href="copyreg.html" title="copyreg — Register pickle support functions"
|
||
accesskey="N">next</a> |</li>
|
||
<li class="right" >
|
||
<a href="persistence.html" title="Data Persistence"
|
||
accesskey="P">previous</a> |</li>
|
||
|
||
<li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"/></li>
|
||
<li><a href="https://www.python.org/">Python</a> »</li>
|
||
<li class="switchers">
|
||
<div class="language_switcher_placeholder"></div>
|
||
<div class="version_switcher_placeholder"></div>
|
||
</li>
|
||
<li>
|
||
|
||
</li>
|
||
<li id="cpython-language-and-version">
|
||
<a href="../index.html">3.13.3 Documentation</a> »
|
||
</li>
|
||
|
||
<li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li>
|
||
<li class="nav-item nav-item-2"><a href="persistence.html" accesskey="U">Data Persistence</a> »</li>
|
||
<li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> — Python object serialization</a></li>
|
||
<li class="right">
|
||
|
||
|
||
<div class="inline-search" role="search">
|
||
<form class="inline-search" action="../search.html" method="get">
|
||
<input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box" />
|
||
<input type="submit" value="Go" />
|
||
</form>
|
||
</div>
|
||
|
|
||
</li>
|
||
<li class="right">
|
||
<label class="theme-selector-label">
|
||
Theme
|
||
<select class="theme-selector" oninput="activateTheme(this.value)">
|
||
<option value="auto" selected>Auto</option>
|
||
<option value="light">Light</option>
|
||
<option value="dark">Dark</option>
|
||
</select>
|
||
</label> |</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
<div class="document">
|
||
<div class="documentwrapper">
|
||
<div class="bodywrapper">
|
||
<div class="body" role="main">
|
||
|
||
<section id="module-pickle">
|
||
<span id="pickle-python-object-serialization"></span><h1><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> — Python object serialization<a class="headerlink" href="#module-pickle" title="Link to this heading">¶</a></h1>
|
||
<p><strong>Source code:</strong> <a class="extlink-source reference external" href="https://github.com/python/cpython/tree/3.13/Lib/pickle.py">Lib/pickle.py</a></p>
|
||
<hr class="docutils" id="index-0" />
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module implements binary protocols for serializing and
|
||
de-serializing a Python object structure. <em>“Pickling”</em> is the process
|
||
whereby a Python object hierarchy is converted into a byte stream, and
|
||
<em>“unpickling”</em> is the inverse operation, whereby a byte stream
|
||
(from a <a class="reference internal" href="../glossary.html#term-binary-file"><span class="xref std std-term">binary file</span></a> or <a class="reference internal" href="../glossary.html#term-bytes-like-object"><span class="xref std std-term">bytes-like object</span></a>) is converted
|
||
back into an object hierarchy. Pickling (and unpickling) is alternatively
|
||
known as “serialization”, “marshalling,” <a class="footnote-reference brackets" href="#id7" id="id1" role="doc-noteref"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></a> or “flattening”; however, to
|
||
avoid confusion, the terms used here are “pickling” and “unpickling”.</p>
|
||
<div class="admonition warning">
|
||
<p class="admonition-title">Warning</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">pickle</span></code> module <strong>is not secure</strong>. Only unpickle data you trust.</p>
|
||
<p>It is possible to construct malicious pickle data which will <strong>execute
|
||
arbitrary code during unpickling</strong>. Never unpickle data that could have come
|
||
from an untrusted source, or that could have been tampered with.</p>
|
||
<p>Consider signing data with <a class="reference internal" href="hmac.html#module-hmac" title="hmac: Keyed-Hashing for Message Authentication (HMAC) implementation"><code class="xref py py-mod docutils literal notranslate"><span class="pre">hmac</span></code></a> if you need to ensure that it has not
|
||
been tampered with.</p>
|
||
<p>Safer serialization formats such as <a class="reference internal" href="json.html#module-json" title="json: Encode and decode the JSON format."><code class="xref py py-mod docutils literal notranslate"><span class="pre">json</span></code></a> may be more appropriate if
|
||
you are processing untrusted data. See <a class="reference internal" href="#comparison-with-json"><span class="std std-ref">Comparison with json</span></a>.</p>
|
||
</div>
|
||
<section id="relationship-to-other-python-modules">
|
||
<h2>Relationship to other Python modules<a class="headerlink" href="#relationship-to-other-python-modules" title="Link to this heading">¶</a></h2>
|
||
<section id="comparison-with-marshal">
|
||
<h3>Comparison with <code class="docutils literal notranslate"><span class="pre">marshal</span></code><a class="headerlink" href="#comparison-with-marshal" title="Link to this heading">¶</a></h3>
|
||
<p>Python has a more primitive serialization module called <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a>, but in
|
||
general <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> should always be the preferred way to serialize Python
|
||
objects. <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> exists primarily to support Python’s <code class="file docutils literal notranslate"><span class="pre">.pyc</span></code>
|
||
files.</p>
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module differs from <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> in several significant ways:</p>
|
||
<ul>
|
||
<li><p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module keeps track of the objects it has already serialized,
|
||
so that later references to the same object won’t be serialized again.
|
||
<a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> doesn’t do this.</p>
|
||
<p>This has implications both for recursive objects and object sharing. Recursive
|
||
objects are objects that contain references to themselves. These are not
|
||
handled by marshal, and in fact, attempting to marshal recursive objects will
|
||
crash your Python interpreter. Object sharing happens when there are multiple
|
||
references to the same object in different places in the object hierarchy being
|
||
serialized. <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> stores such objects only once, and ensures that all
|
||
other references point to the master copy. Shared objects remain shared, which
|
||
can be very important for mutable objects.</p>
|
||
</li>
|
||
<li><p><a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> cannot be used to serialize user-defined classes and their
|
||
instances. <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> can save and restore class instances transparently,
|
||
however the class definition must be importable and live in the same module as
|
||
when the object was stored.</p></li>
|
||
<li><p>The <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> serialization format is not guaranteed to be portable
|
||
across Python versions. Because its primary job in life is to support
|
||
<code class="file docutils literal notranslate"><span class="pre">.pyc</span></code> files, the Python implementers reserve the right to change the
|
||
serialization format in non-backwards compatible ways should the need arise.
|
||
The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> serialization format is guaranteed to be backwards compatible
|
||
across Python releases provided a compatible pickle protocol is chosen and
|
||
pickling and unpickling code deals with Python 2 to Python 3 type differences
|
||
if your data is crossing that unique breaking change language boundary.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="comparison-with-json">
|
||
<span id="id2"></span><h3>Comparison with <code class="docutils literal notranslate"><span class="pre">json</span></code><a class="headerlink" href="#comparison-with-json" title="Link to this heading">¶</a></h3>
|
||
<p>There are fundamental differences between the pickle protocols and
|
||
<a class="reference external" href="https://json.org">JSON (JavaScript Object Notation)</a>:</p>
|
||
<ul class="simple">
|
||
<li><p>JSON is a text serialization format (it outputs unicode text, although
|
||
most of the time it is then encoded to <code class="docutils literal notranslate"><span class="pre">utf-8</span></code>), while pickle is
|
||
a binary serialization format;</p></li>
|
||
<li><p>JSON is human-readable, while pickle is not;</p></li>
|
||
<li><p>JSON is interoperable and widely used outside of the Python ecosystem,
|
||
while pickle is Python-specific;</p></li>
|
||
<li><p>JSON, by default, can only represent a subset of the Python built-in
|
||
types, and no custom classes; pickle can represent an extremely large
|
||
number of Python types (many of them automatically, by clever usage
|
||
of Python’s introspection facilities; complex cases can be tackled by
|
||
implementing <a class="reference internal" href="#pickle-inst"><span class="std std-ref">specific object APIs</span></a>);</p></li>
|
||
<li><p>Unlike pickle, deserializing untrusted JSON does not in itself create an
|
||
arbitrary code execution vulnerability.</p></li>
|
||
</ul>
|
||
<div class="admonition seealso">
|
||
<p class="admonition-title">See also</p>
|
||
<p>The <a class="reference internal" href="json.html#module-json" title="json: Encode and decode the JSON format."><code class="xref py py-mod docutils literal notranslate"><span class="pre">json</span></code></a> module: a standard library module allowing JSON
|
||
serialization and deserialization.</p>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="data-stream-format">
|
||
<span id="pickle-protocols"></span><h2>Data stream format<a class="headerlink" href="#data-stream-format" title="Link to this heading">¶</a></h2>
|
||
<p id="index-1">The data format used by <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> is Python-specific. This has the
|
||
advantage that there are no restrictions imposed by external standards such as
|
||
JSON (which can’t represent pointer sharing); however it means that
|
||
non-Python programs may not be able to reconstruct pickled Python objects.</p>
|
||
<p>By default, the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> data format uses a relatively compact binary
|
||
representation. If you need optimal size characteristics, you can efficiently
|
||
<a class="reference internal" href="archiving.html"><span class="doc">compress</span></a> pickled data.</p>
|
||
<p>The module <a class="reference internal" href="pickletools.html#module-pickletools" title="pickletools: Contains extensive comments about the pickle protocols and pickle-machine opcodes, as well as some useful functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickletools</span></code></a> contains tools for analyzing data streams
|
||
generated by <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a>. <a class="reference internal" href="pickletools.html#module-pickletools" title="pickletools: Contains extensive comments about the pickle protocols and pickle-machine opcodes, as well as some useful functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickletools</span></code></a> source code has extensive
|
||
comments about opcodes used by pickle protocols.</p>
|
||
<p>There are currently 6 different protocols which can be used for pickling.
|
||
The higher the protocol used, the more recent the version of Python needed
|
||
to read the pickle produced.</p>
|
||
<ul class="simple">
|
||
<li><p>Protocol version 0 is the original “human-readable” protocol and is
|
||
backwards compatible with earlier versions of Python.</p></li>
|
||
<li><p>Protocol version 1 is an old binary format which is also compatible with
|
||
earlier versions of Python.</p></li>
|
||
<li><p>Protocol version 2 was introduced in Python 2.3. It provides much more
|
||
efficient pickling of <a class="reference internal" href="../glossary.html#term-new-style-class"><span class="xref std std-term">new-style classes</span></a>. Refer to <span class="target" id="index-2"></span><a class="pep reference external" href="https://peps.python.org/pep-0307/"><strong>PEP 307</strong></a> for
|
||
information about improvements brought by protocol 2.</p></li>
|
||
<li><p>Protocol version 3 was added in Python 3.0. It has explicit support for
|
||
<a class="reference internal" href="stdtypes.html#bytes" title="bytes"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytes</span></code></a> objects and cannot be unpickled by Python 2.x. This was
|
||
the default protocol in Python 3.0–3.7.</p></li>
|
||
<li><p>Protocol version 4 was added in Python 3.4. It adds support for very large
|
||
objects, pickling more kinds of objects, and some data format
|
||
optimizations. It is the default protocol starting with Python 3.8.
|
||
Refer to <span class="target" id="index-3"></span><a class="pep reference external" href="https://peps.python.org/pep-3154/"><strong>PEP 3154</strong></a> for information about improvements brought by
|
||
protocol 4.</p></li>
|
||
<li><p>Protocol version 5 was added in Python 3.8. It adds support for out-of-band
|
||
data and speedup for in-band data. Refer to <span class="target" id="index-4"></span><a class="pep reference external" href="https://peps.python.org/pep-0574/"><strong>PEP 574</strong></a> for information about
|
||
improvements brought by protocol 5.</p></li>
|
||
</ul>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Serialization is a more primitive notion than persistence; although
|
||
<a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> reads and writes file objects, it does not handle the issue of
|
||
naming persistent objects, nor the (even more complicated) issue of concurrent
|
||
access to persistent objects. The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module can transform a complex
|
||
object into a byte stream and it can transform the byte stream into an object
|
||
with the same internal structure. Perhaps the most obvious thing to do with
|
||
these byte streams is to write them onto a file, but it is also conceivable to
|
||
send them across a network or store them in a database. The <a class="reference internal" href="shelve.html#module-shelve" title="shelve: Python object persistence."><code class="xref py py-mod docutils literal notranslate"><span class="pre">shelve</span></code></a>
|
||
module provides a simple interface to pickle and unpickle objects on
|
||
DBM-style database files.</p>
|
||
</div>
|
||
</section>
|
||
<section id="module-interface">
|
||
<h2>Module Interface<a class="headerlink" href="#module-interface" title="Link to this heading">¶</a></h2>
|
||
<p>To serialize an object hierarchy, you simply call the <a class="reference internal" href="#pickle.dumps" title="pickle.dumps"><code class="xref py py-func docutils literal notranslate"><span class="pre">dumps()</span></code></a> function.
|
||
Similarly, to de-serialize a data stream, you call the <a class="reference internal" href="#pickle.loads" title="pickle.loads"><code class="xref py py-func docutils literal notranslate"><span class="pre">loads()</span></code></a> function.
|
||
However, if you want more control over serialization and de-serialization,
|
||
you can create a <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> or an <a class="reference internal" href="#pickle.Unpickler" title="pickle.Unpickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code></a> object, respectively.</p>
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module provides the following constants:</p>
|
||
<dl class="py data">
|
||
<dt class="sig sig-object py" id="pickle.HIGHEST_PROTOCOL">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">HIGHEST_PROTOCOL</span></span><a class="headerlink" href="#pickle.HIGHEST_PROTOCOL" title="Link to this definition">¶</a></dt>
|
||
<dd><p>An integer, the highest <a class="reference internal" href="#pickle-protocols"><span class="std std-ref">protocol version</span></a>
|
||
available. This value can be passed as a <em>protocol</em> value to functions
|
||
<a class="reference internal" href="#pickle.dump" title="pickle.dump"><code class="xref py py-func docutils literal notranslate"><span class="pre">dump()</span></code></a> and <a class="reference internal" href="#pickle.dumps" title="pickle.dumps"><code class="xref py py-func docutils literal notranslate"><span class="pre">dumps()</span></code></a> as well as the <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a>
|
||
constructor.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py data">
|
||
<dt class="sig sig-object py" id="pickle.DEFAULT_PROTOCOL">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">DEFAULT_PROTOCOL</span></span><a class="headerlink" href="#pickle.DEFAULT_PROTOCOL" title="Link to this definition">¶</a></dt>
|
||
<dd><p>An integer, the default <a class="reference internal" href="#pickle-protocols"><span class="std std-ref">protocol version</span></a> used
|
||
for pickling. May be less than <a class="reference internal" href="#pickle.HIGHEST_PROTOCOL" title="pickle.HIGHEST_PROTOCOL"><code class="xref py py-data docutils literal notranslate"><span class="pre">HIGHEST_PROTOCOL</span></code></a>. Currently the
|
||
default protocol is 4, first introduced in Python 3.4 and incompatible
|
||
with previous versions.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.0: </span>The default protocol is 3.</p>
|
||
</div>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The default protocol is 4.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module provides the following functions to make the pickling
|
||
process more convenient:</p>
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="pickle.dump">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">dump</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">protocol</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_callback</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.dump" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Write the pickled representation of the object <em>obj</em> to the open
|
||
<a class="reference internal" href="../glossary.html#term-file-object"><span class="xref std std-term">file object</span></a> <em>file</em>. This is equivalent to
|
||
<code class="docutils literal notranslate"><span class="pre">Pickler(file,</span> <span class="pre">protocol).dump(obj)</span></code>.</p>
|
||
<p>Arguments <em>file</em>, <em>protocol</em>, <em>fix_imports</em> and <em>buffer_callback</em> have
|
||
the same meaning as in the <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> constructor.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffer_callback</em> argument was added.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="pickle.dumps">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">dumps</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">protocol</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_callback</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.dumps" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Return the pickled representation of the object <em>obj</em> as a <a class="reference internal" href="stdtypes.html#bytes" title="bytes"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytes</span></code></a> object,
|
||
instead of writing it to a file.</p>
|
||
<p>Arguments <em>protocol</em>, <em>fix_imports</em> and <em>buffer_callback</em> have the same
|
||
meaning as in the <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> constructor.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffer_callback</em> argument was added.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="pickle.load">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">load</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'ASCII'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">errors</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'strict'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.load" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Read the pickled representation of an object from the open <a class="reference internal" href="../glossary.html#term-file-object"><span class="xref std std-term">file object</span></a>
|
||
<em>file</em> and return the reconstituted object hierarchy specified therein.
|
||
This is equivalent to <code class="docutils literal notranslate"><span class="pre">Unpickler(file).load()</span></code>.</p>
|
||
<p>The protocol version of the pickle is detected automatically, so no
|
||
protocol argument is needed. Bytes past the pickled representation
|
||
of the object are ignored.</p>
|
||
<p>Arguments <em>file</em>, <em>fix_imports</em>, <em>encoding</em>, <em>errors</em>, <em>strict</em> and <em>buffers</em>
|
||
have the same meaning as in the <a class="reference internal" href="#pickle.Unpickler" title="pickle.Unpickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code></a> constructor.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffers</em> argument was added.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="pickle.loads">
|
||
<span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">loads</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span></em>, <em class="sig-param"><span class="positional-only-separator o"><abbr title="Positional-only parameter separator (PEP 570)"><span class="pre">/</span></abbr></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'ASCII'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">errors</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'strict'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.loads" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Return the reconstituted object hierarchy of the pickled representation
|
||
<em>data</em> of an object. <em>data</em> must be a <a class="reference internal" href="../glossary.html#term-bytes-like-object"><span class="xref std std-term">bytes-like object</span></a>.</p>
|
||
<p>The protocol version of the pickle is detected automatically, so no
|
||
protocol argument is needed. Bytes past the pickled representation
|
||
of the object are ignored.</p>
|
||
<p>Arguments <em>fix_imports</em>, <em>encoding</em>, <em>errors</em>, <em>strict</em> and <em>buffers</em>
|
||
have the same meaning as in the <a class="reference internal" href="#pickle.Unpickler" title="pickle.Unpickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code></a> constructor.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffers</em> argument was added.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module defines three exceptions:</p>
|
||
<dl class="py exception">
|
||
<dt class="sig sig-object py" id="pickle.PickleError">
|
||
<em class="property"><span class="k"><span class="pre">exception</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">PickleError</span></span><a class="headerlink" href="#pickle.PickleError" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Common base class for the other pickling exceptions. It inherits from
|
||
<a class="reference internal" href="exceptions.html#Exception" title="Exception"><code class="xref py py-exc docutils literal notranslate"><span class="pre">Exception</span></code></a>.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py exception">
|
||
<dt class="sig sig-object py" id="pickle.PicklingError">
|
||
<em class="property"><span class="k"><span class="pre">exception</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">PicklingError</span></span><a class="headerlink" href="#pickle.PicklingError" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Error raised when an unpicklable object is encountered by <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a>.
|
||
It inherits from <a class="reference internal" href="#pickle.PickleError" title="pickle.PickleError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">PickleError</span></code></a>.</p>
|
||
<p>Refer to <a class="reference internal" href="#pickle-picklable"><span class="std std-ref">What can be pickled and unpickled?</span></a> to learn what kinds of objects can be
|
||
pickled.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py exception">
|
||
<dt class="sig sig-object py" id="pickle.UnpicklingError">
|
||
<em class="property"><span class="k"><span class="pre">exception</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">UnpicklingError</span></span><a class="headerlink" href="#pickle.UnpicklingError" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Error raised when there is a problem unpickling an object, such as a data
|
||
corruption or a security violation. It inherits from <a class="reference internal" href="#pickle.PickleError" title="pickle.PickleError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">PickleError</span></code></a>.</p>
|
||
<p>Note that other exceptions may also be raised during unpickling, including
|
||
(but not necessarily limited to) AttributeError, EOFError, ImportError, and
|
||
IndexError.</p>
|
||
</dd></dl>
|
||
|
||
<p>The <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module exports three classes, <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a>,
|
||
<a class="reference internal" href="#pickle.Unpickler" title="pickle.Unpickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code></a> and <a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a>:</p>
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="pickle.Pickler">
|
||
<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">Pickler</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">protocol</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffer_callback</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Pickler" title="Link to this definition">¶</a></dt>
|
||
<dd><p>This takes a binary file for writing a pickle data stream.</p>
|
||
<p>The optional <em>protocol</em> argument, an integer, tells the pickler to use
|
||
the given protocol; supported protocols are 0 to <a class="reference internal" href="#pickle.HIGHEST_PROTOCOL" title="pickle.HIGHEST_PROTOCOL"><code class="xref py py-data docutils literal notranslate"><span class="pre">HIGHEST_PROTOCOL</span></code></a>.
|
||
If not specified, the default is <a class="reference internal" href="#pickle.DEFAULT_PROTOCOL" title="pickle.DEFAULT_PROTOCOL"><code class="xref py py-data docutils literal notranslate"><span class="pre">DEFAULT_PROTOCOL</span></code></a>. If a negative
|
||
number is specified, <a class="reference internal" href="#pickle.HIGHEST_PROTOCOL" title="pickle.HIGHEST_PROTOCOL"><code class="xref py py-data docutils literal notranslate"><span class="pre">HIGHEST_PROTOCOL</span></code></a> is selected.</p>
|
||
<p>The <em>file</em> argument must have a write() method that accepts a single bytes
|
||
argument. It can thus be an on-disk file opened for binary writing, an
|
||
<a class="reference internal" href="io.html#io.BytesIO" title="io.BytesIO"><code class="xref py py-class docutils literal notranslate"><span class="pre">io.BytesIO</span></code></a> instance, or any other custom object that meets this
|
||
interface.</p>
|
||
<p>If <em>fix_imports</em> is true and <em>protocol</em> is less than 3, pickle will try to
|
||
map the new Python 3 names to the old module names used in Python 2, so
|
||
that the pickle data stream is readable with Python 2.</p>
|
||
<p>If <em>buffer_callback</em> is <code class="docutils literal notranslate"><span class="pre">None</span></code> (the default), buffer views are
|
||
serialized into <em>file</em> as part of the pickle stream.</p>
|
||
<p>If <em>buffer_callback</em> is not <code class="docutils literal notranslate"><span class="pre">None</span></code>, then it can be called any number
|
||
of times with a buffer view. If the callback returns a false value
|
||
(such as <code class="docutils literal notranslate"><span class="pre">None</span></code>), the given buffer is <a class="reference internal" href="#pickle-oob"><span class="std std-ref">out-of-band</span></a>;
|
||
otherwise the buffer is serialized in-band, i.e. inside the pickle stream.</p>
|
||
<p>It is an error if <em>buffer_callback</em> is not <code class="docutils literal notranslate"><span class="pre">None</span></code> and <em>protocol</em> is
|
||
<code class="docutils literal notranslate"><span class="pre">None</span></code> or smaller than 5.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffer_callback</em> argument was added.</p>
|
||
</div>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Pickler.dump">
|
||
<span class="sig-name descname"><span class="pre">dump</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Pickler.dump" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Write the pickled representation of <em>obj</em> to the open file object given in
|
||
the constructor.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Pickler.persistent_id">
|
||
<span class="sig-name descname"><span class="pre">persistent_id</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Pickler.persistent_id" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Do nothing by default. This exists so a subclass can override it.</p>
|
||
<p>If <a class="reference internal" href="#pickle.Pickler.persistent_id" title="pickle.Pickler.persistent_id"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_id()</span></code></a> returns <code class="docutils literal notranslate"><span class="pre">None</span></code>, <em>obj</em> is pickled as usual. Any
|
||
other value causes <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> to emit the returned value as a
|
||
persistent ID for <em>obj</em>. The meaning of this persistent ID should be
|
||
defined by <a class="reference internal" href="#pickle.Unpickler.persistent_load" title="pickle.Unpickler.persistent_load"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Unpickler.persistent_load()</span></code></a>. Note that the value
|
||
returned by <a class="reference internal" href="#pickle.Pickler.persistent_id" title="pickle.Pickler.persistent_id"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_id()</span></code></a> cannot itself have a persistent ID.</p>
|
||
<p>See <a class="reference internal" href="#pickle-persistent"><span class="std std-ref">Persistence of External Objects</span></a> for details and examples of uses.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.13: </span>Add the default implementation of this method in the C implementation
|
||
of <code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code>.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="pickle.Pickler.dispatch_table">
|
||
<span class="sig-name descname"><span class="pre">dispatch_table</span></span><a class="headerlink" href="#pickle.Pickler.dispatch_table" title="Link to this definition">¶</a></dt>
|
||
<dd><p>A pickler object’s dispatch table is a registry of <em>reduction
|
||
functions</em> of the kind which can be declared using
|
||
<a class="reference internal" href="copyreg.html#copyreg.pickle" title="copyreg.pickle"><code class="xref py py-func docutils literal notranslate"><span class="pre">copyreg.pickle()</span></code></a>. It is a mapping whose keys are classes
|
||
and whose values are reduction functions. A reduction function
|
||
takes a single argument of the associated class and should
|
||
conform to the same interface as a <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a>
|
||
method.</p>
|
||
<p>By default, a pickler object will not have a
|
||
<a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a> attribute, and it will instead use the
|
||
global dispatch table managed by the <a class="reference internal" href="copyreg.html#module-copyreg" title="copyreg: Register pickle support functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code></a> module.
|
||
However, to customize the pickling for a specific pickler object
|
||
one can set the <a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a> attribute to a dict-like
|
||
object. Alternatively, if a subclass of <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> has a
|
||
<a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a> attribute then this will be used as the
|
||
default dispatch table for instances of that class.</p>
|
||
<p>See <a class="reference internal" href="#pickle-dispatch"><span class="std std-ref">Dispatch Tables</span></a> for usage examples.</p>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.3.</span></p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Pickler.reducer_override">
|
||
<span class="sig-name descname"><span class="pre">reducer_override</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">obj</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Pickler.reducer_override" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Special reducer that can be defined in <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> subclasses. This
|
||
method has priority over any reducer in the <a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a>. It
|
||
should conform to the same interface as a <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> method, and
|
||
can optionally return <a class="reference internal" href="constants.html#NotImplemented" title="NotImplemented"><code class="xref py py-data docutils literal notranslate"><span class="pre">NotImplemented</span></code></a> to fallback on
|
||
<a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a>-registered reducers to pickle <code class="docutils literal notranslate"><span class="pre">obj</span></code>.</p>
|
||
<p>For a detailed example, see <a class="reference internal" href="#reducer-override"><span class="std std-ref">Custom Reduction for Types, Functions, and Other Objects</span></a>.</p>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.8.</span></p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="pickle.Pickler.fast">
|
||
<span class="sig-name descname"><span class="pre">fast</span></span><a class="headerlink" href="#pickle.Pickler.fast" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Deprecated. Enable fast mode if set to a true value. The fast mode
|
||
disables the usage of memo, therefore speeding the pickling process by not
|
||
generating superfluous PUT opcodes. It should not be used with
|
||
self-referential objects, doing otherwise will cause <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> to
|
||
recurse infinitely.</p>
|
||
<p>Use <a class="reference internal" href="pickletools.html#pickletools.optimize" title="pickletools.optimize"><code class="xref py py-func docutils literal notranslate"><span class="pre">pickletools.optimize()</span></code></a> if you need more compact pickles.</p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="pickle.Unpickler">
|
||
<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">Unpickler</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">file</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">fix_imports</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'ASCII'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">errors</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'strict'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">buffers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Unpickler" title="Link to this definition">¶</a></dt>
|
||
<dd><p>This takes a binary file for reading a pickle data stream.</p>
|
||
<p>The protocol version of the pickle is detected automatically, so no
|
||
protocol argument is needed.</p>
|
||
<p>The argument <em>file</em> must have three methods, a read() method that takes an
|
||
integer argument, a readinto() method that takes a buffer argument
|
||
and a readline() method that requires no arguments, as in the
|
||
<a class="reference internal" href="io.html#io.BufferedIOBase" title="io.BufferedIOBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">io.BufferedIOBase</span></code></a> interface. Thus <em>file</em> can be an on-disk file
|
||
opened for binary reading, an <a class="reference internal" href="io.html#io.BytesIO" title="io.BytesIO"><code class="xref py py-class docutils literal notranslate"><span class="pre">io.BytesIO</span></code></a> object, or any other
|
||
custom object that meets this interface.</p>
|
||
<p>The optional arguments <em>fix_imports</em>, <em>encoding</em> and <em>errors</em> are used
|
||
to control compatibility support for pickle stream generated by Python 2.
|
||
If <em>fix_imports</em> is true, pickle will try to map the old Python 2 names
|
||
to the new names used in Python 3. The <em>encoding</em> and <em>errors</em> tell
|
||
pickle how to decode 8-bit string instances pickled by Python 2;
|
||
these default to ‘ASCII’ and ‘strict’, respectively. The <em>encoding</em> can
|
||
be ‘bytes’ to read these 8-bit string instances as bytes objects.
|
||
Using <code class="docutils literal notranslate"><span class="pre">encoding='latin1'</span></code> is required for unpickling NumPy arrays and
|
||
instances of <a class="reference internal" href="datetime.html#datetime.datetime" title="datetime.datetime"><code class="xref py py-class docutils literal notranslate"><span class="pre">datetime</span></code></a>, <a class="reference internal" href="datetime.html#datetime.date" title="datetime.date"><code class="xref py py-class docutils literal notranslate"><span class="pre">date</span></code></a> and
|
||
<a class="reference internal" href="datetime.html#datetime.time" title="datetime.time"><code class="xref py py-class docutils literal notranslate"><span class="pre">time</span></code></a> pickled by Python 2.</p>
|
||
<p>If <em>buffers</em> is <code class="docutils literal notranslate"><span class="pre">None</span></code> (the default), then all data necessary for
|
||
deserialization must be contained in the pickle stream. This means
|
||
that the <em>buffer_callback</em> argument was <code class="docutils literal notranslate"><span class="pre">None</span></code> when a <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a>
|
||
was instantiated (or when <a class="reference internal" href="#pickle.dump" title="pickle.dump"><code class="xref py py-func docutils literal notranslate"><span class="pre">dump()</span></code></a> or <a class="reference internal" href="#pickle.dumps" title="pickle.dumps"><code class="xref py py-func docutils literal notranslate"><span class="pre">dumps()</span></code></a> was called).</p>
|
||
<p>If <em>buffers</em> is not <code class="docutils literal notranslate"><span class="pre">None</span></code>, it should be an iterable of buffer-enabled
|
||
objects that is consumed each time the pickle stream references
|
||
an <a class="reference internal" href="#pickle-oob"><span class="std std-ref">out-of-band</span></a> buffer view. Such buffers have been
|
||
given in order to the <em>buffer_callback</em> of a Pickler object.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.8: </span>The <em>buffers</em> argument was added.</p>
|
||
</div>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Unpickler.load">
|
||
<span class="sig-name descname"><span class="pre">load</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Unpickler.load" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Read the pickled representation of an object from the open file object
|
||
given in the constructor, and return the reconstituted object hierarchy
|
||
specified therein. Bytes past the pickled representation of the object
|
||
are ignored.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Unpickler.persistent_load">
|
||
<span class="sig-name descname"><span class="pre">persistent_load</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">pid</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Unpickler.persistent_load" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Raise an <a class="reference internal" href="#pickle.UnpicklingError" title="pickle.UnpicklingError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">UnpicklingError</span></code></a> by default.</p>
|
||
<p>If defined, <a class="reference internal" href="#pickle.Unpickler.persistent_load" title="pickle.Unpickler.persistent_load"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_load()</span></code></a> should return the object specified by
|
||
the persistent ID <em>pid</em>. If an invalid persistent ID is encountered, an
|
||
<a class="reference internal" href="#pickle.UnpicklingError" title="pickle.UnpicklingError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">UnpicklingError</span></code></a> should be raised.</p>
|
||
<p>See <a class="reference internal" href="#pickle-persistent"><span class="std std-ref">Persistence of External Objects</span></a> for details and examples of uses.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.13: </span>Add the default implementation of this method in the C implementation
|
||
of <code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code>.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.Unpickler.find_class">
|
||
<span class="sig-name descname"><span class="pre">find_class</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">module</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.Unpickler.find_class" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Import <em>module</em> if necessary and return the object called <em>name</em> from it,
|
||
where the <em>module</em> and <em>name</em> arguments are <a class="reference internal" href="stdtypes.html#str" title="str"><code class="xref py py-class docutils literal notranslate"><span class="pre">str</span></code></a> objects. Note,
|
||
unlike its name suggests, <a class="reference internal" href="#pickle.Unpickler.find_class" title="pickle.Unpickler.find_class"><code class="xref py py-meth docutils literal notranslate"><span class="pre">find_class()</span></code></a> is also used for finding
|
||
functions.</p>
|
||
<p>Subclasses may override this to gain control over what type of objects and
|
||
how they can be loaded, potentially reducing security risks. Refer to
|
||
<a class="reference internal" href="#pickle-restrict"><span class="std std-ref">Restricting Globals</span></a> for details.</p>
|
||
<p class="audit-hook">Raises an <a class="reference internal" href="sys.html#auditing"><span class="std std-ref">auditing event</span></a> <code class="docutils literal notranslate"><span class="pre">pickle.find_class</span></code> with arguments <code class="docutils literal notranslate"><span class="pre">module</span></code>, <code class="docutils literal notranslate"><span class="pre">name</span></code>.</p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="pickle.PickleBuffer">
|
||
<em class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">pickle.</span></span><span class="sig-name descname"><span class="pre">PickleBuffer</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">buffer</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#pickle.PickleBuffer" title="Link to this definition">¶</a></dt>
|
||
<dd><p>A wrapper for a buffer representing picklable data. <em>buffer</em> must be a
|
||
<a class="reference internal" href="../c-api/buffer.html#bufferobjects"><span class="std std-ref">buffer-providing</span></a> object, such as a
|
||
<a class="reference internal" href="../glossary.html#term-bytes-like-object"><span class="xref std std-term">bytes-like object</span></a> or a N-dimensional array.</p>
|
||
<p><a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> is itself a buffer provider, therefore it is
|
||
possible to pass it to other APIs expecting a buffer-providing object,
|
||
such as <a class="reference internal" href="stdtypes.html#memoryview" title="memoryview"><code class="xref py py-class docutils literal notranslate"><span class="pre">memoryview</span></code></a>.</p>
|
||
<p><a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> objects can only be serialized using pickle
|
||
protocol 5 or higher. They are eligible for
|
||
<a class="reference internal" href="#pickle-oob"><span class="std std-ref">out-of-band serialization</span></a>.</p>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.8.</span></p>
|
||
</div>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.PickleBuffer.raw">
|
||
<span class="sig-name descname"><span class="pre">raw</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#pickle.PickleBuffer.raw" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Return a <a class="reference internal" href="stdtypes.html#memoryview" title="memoryview"><code class="xref py py-class docutils literal notranslate"><span class="pre">memoryview</span></code></a> of the memory area underlying this buffer.
|
||
The returned object is a one-dimensional, C-contiguous memoryview
|
||
with format <code class="docutils literal notranslate"><span class="pre">B</span></code> (unsigned bytes). <a class="reference internal" href="exceptions.html#BufferError" title="BufferError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">BufferError</span></code></a> is raised if
|
||
the buffer is neither C- nor Fortran-contiguous.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="pickle.PickleBuffer.release">
|
||
<span class="sig-name descname"><span class="pre">release</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#pickle.PickleBuffer.release" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Release the underlying buffer exposed by the PickleBuffer object.</p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="what-can-be-pickled-and-unpickled">
|
||
<span id="pickle-picklable"></span><h2>What can be pickled and unpickled?<a class="headerlink" href="#what-can-be-pickled-and-unpickled" title="Link to this heading">¶</a></h2>
|
||
<p>The following types can be pickled:</p>
|
||
<ul class="simple">
|
||
<li><p>built-in constants (<code class="docutils literal notranslate"><span class="pre">None</span></code>, <code class="docutils literal notranslate"><span class="pre">True</span></code>, <code class="docutils literal notranslate"><span class="pre">False</span></code>, <code class="docutils literal notranslate"><span class="pre">Ellipsis</span></code>, and
|
||
<a class="reference internal" href="constants.html#NotImplemented" title="NotImplemented"><code class="xref py py-data docutils literal notranslate"><span class="pre">NotImplemented</span></code></a>);</p></li>
|
||
<li><p>integers, floating-point numbers, complex numbers;</p></li>
|
||
<li><p>strings, bytes, bytearrays;</p></li>
|
||
<li><p>tuples, lists, sets, and dictionaries containing only picklable objects;</p></li>
|
||
<li><p>functions (built-in and user-defined) accessible from the top level of a
|
||
module (using <a class="reference internal" href="../reference/compound_stmts.html#def"><code class="xref std std-keyword docutils literal notranslate"><span class="pre">def</span></code></a>, not <a class="reference internal" href="../reference/expressions.html#lambda"><code class="xref std std-keyword docutils literal notranslate"><span class="pre">lambda</span></code></a>);</p></li>
|
||
<li><p>classes accessible from the top level of a module;</p></li>
|
||
<li><p>instances of such classes whose the result of calling <a class="reference internal" href="#object.__getstate__" title="object.__getstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getstate__()</span></code></a>
|
||
is picklable (see section <a class="reference internal" href="#pickle-inst"><span class="std std-ref">Pickling Class Instances</span></a> for details).</p></li>
|
||
</ul>
|
||
<p>Attempts to pickle unpicklable objects will raise the <a class="reference internal" href="#pickle.PicklingError" title="pickle.PicklingError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">PicklingError</span></code></a>
|
||
exception; when this happens, an unspecified number of bytes may have already
|
||
been written to the underlying file. Trying to pickle a highly recursive data
|
||
structure may exceed the maximum recursion depth, a <a class="reference internal" href="exceptions.html#RecursionError" title="RecursionError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">RecursionError</span></code></a> will be
|
||
raised in this case. You can carefully raise this limit with
|
||
<a class="reference internal" href="sys.html#sys.setrecursionlimit" title="sys.setrecursionlimit"><code class="xref py py-func docutils literal notranslate"><span class="pre">sys.setrecursionlimit()</span></code></a>.</p>
|
||
<p>Note that functions (built-in and user-defined) are pickled by fully
|
||
<a class="reference internal" href="../glossary.html#term-qualified-name"><span class="xref std std-term">qualified name</span></a>, not by value. <a class="footnote-reference brackets" href="#id8" id="id3" role="doc-noteref"><span class="fn-bracket">[</span>2<span class="fn-bracket">]</span></a> This means that only the function name is
|
||
pickled, along with the name of the containing module and classes. Neither
|
||
the function’s code, nor any of its function attributes are pickled. Thus the
|
||
defining module must be importable in the unpickling environment, and the module
|
||
must contain the named object, otherwise an exception will be raised. <a class="footnote-reference brackets" href="#id9" id="id4" role="doc-noteref"><span class="fn-bracket">[</span>3<span class="fn-bracket">]</span></a></p>
|
||
<p>Similarly, classes are pickled by fully qualified name, so the same restrictions in
|
||
the unpickling environment apply. Note that none of the class’s code or data is
|
||
pickled, so in the following example the class attribute <code class="docutils literal notranslate"><span class="pre">attr</span></code> is not
|
||
restored in the unpickling environment:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">Foo</span><span class="p">:</span>
|
||
<span class="n">attr</span> <span class="o">=</span> <span class="s1">'A class attribute'</span>
|
||
|
||
<span class="n">picklestring</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">Foo</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>These restrictions are why picklable functions and classes must be defined at
|
||
the top level of a module.</p>
|
||
<p>Similarly, when class instances are pickled, their class’s code and data are not
|
||
pickled along with them. Only the instance data are pickled. This is done on
|
||
purpose, so you can fix bugs in a class or add methods to the class and still
|
||
load objects that were created with an earlier version of the class. If you
|
||
plan to have long-lived objects that will see many versions of a class, it may
|
||
be worthwhile to put a version number in the objects so that suitable
|
||
conversions can be made by the class’s <a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a> method.</p>
|
||
</section>
|
||
<section id="pickling-class-instances">
|
||
<span id="pickle-inst"></span><h2>Pickling Class Instances<a class="headerlink" href="#pickling-class-instances" title="Link to this heading">¶</a></h2>
|
||
<p>In this section, we describe the general mechanisms available to you to define,
|
||
customize, and control how class instances are pickled and unpickled.</p>
|
||
<p>In most cases, no additional code is needed to make instances picklable. By
|
||
default, pickle will retrieve the class and the attributes of an instance via
|
||
introspection. When a class instance is unpickled, its <a class="reference internal" href="../reference/datamodel.html#object.__init__" title="object.__init__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__init__()</span></code></a> method
|
||
is usually <em>not</em> invoked. The default behaviour first creates an uninitialized
|
||
instance and then restores the saved attributes. The following code shows an
|
||
implementation of this behaviour:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="vm">__class__</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">restore</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">attributes</span><span class="p">):</span>
|
||
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span>
|
||
<span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">attributes</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">obj</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Classes can alter the default behaviour by providing one or several special
|
||
methods:</p>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__getnewargs_ex__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__getnewargs_ex__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#object.__getnewargs_ex__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>In protocols 2 and newer, classes that implements the
|
||
<a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a> method can dictate the values passed to the
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__new__" title="object.__new__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__new__()</span></code></a> method upon unpickling. The method must return a pair
|
||
<code class="docutils literal notranslate"><span class="pre">(args,</span> <span class="pre">kwargs)</span></code> where <em>args</em> is a tuple of positional arguments
|
||
and <em>kwargs</em> a dictionary of named arguments for constructing the
|
||
object. Those will be passed to the <a class="reference internal" href="../reference/datamodel.html#object.__new__" title="object.__new__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__new__()</span></code></a> method upon
|
||
unpickling.</p>
|
||
<p>You should implement this method if the <a class="reference internal" href="../reference/datamodel.html#object.__new__" title="object.__new__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__new__()</span></code></a> method of your
|
||
class requires keyword-only arguments. Otherwise, it is recommended for
|
||
compatibility to implement <a class="reference internal" href="#object.__getnewargs__" title="object.__getnewargs__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs__()</span></code></a>.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.6: </span><a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a> is now used in protocols 2 and 3.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__getnewargs__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__getnewargs__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#object.__getnewargs__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>This method serves a similar purpose as <a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a>, but
|
||
supports only positional arguments. It must return a tuple of arguments
|
||
<code class="docutils literal notranslate"><span class="pre">args</span></code> which will be passed to the <a class="reference internal" href="../reference/datamodel.html#object.__new__" title="object.__new__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__new__()</span></code></a> method upon unpickling.</p>
|
||
<p><a class="reference internal" href="#object.__getnewargs__" title="object.__getnewargs__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs__()</span></code></a> will not be called if <a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a> is
|
||
defined.</p>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.6: </span>Before Python 3.6, <a class="reference internal" href="#object.__getnewargs__" title="object.__getnewargs__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs__()</span></code></a> was called instead of
|
||
<a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a> in protocols 2 and 3.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__getstate__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__getstate__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#object.__getstate__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Classes can further influence how their instances are pickled by overriding
|
||
the method <a class="reference internal" href="#object.__getstate__" title="object.__getstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getstate__()</span></code></a>. It is called and the returned object
|
||
is pickled as the contents for the instance, instead of a default state.
|
||
There are several cases:</p>
|
||
<ul class="simple">
|
||
<li><p>For a class that has no instance <a class="reference internal" href="../reference/datamodel.html#object.__dict__" title="object.__dict__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__dict__</span></code></a> and no
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__slots__" title="object.__slots__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__slots__</span></code></a>, the default state is <code class="docutils literal notranslate"><span class="pre">None</span></code>.</p></li>
|
||
<li><p>For a class that has an instance <a class="reference internal" href="../reference/datamodel.html#object.__dict__" title="object.__dict__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__dict__</span></code></a> and no
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__slots__" title="object.__slots__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__slots__</span></code></a>, the default state is <code class="docutils literal notranslate"><span class="pre">self.__dict__</span></code>.</p></li>
|
||
<li><p>For a class that has an instance <a class="reference internal" href="../reference/datamodel.html#object.__dict__" title="object.__dict__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__dict__</span></code></a> and
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__slots__" title="object.__slots__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__slots__</span></code></a>, the default state is a tuple consisting of two
|
||
dictionaries: <code class="docutils literal notranslate"><span class="pre">self.__dict__</span></code>, and a dictionary mapping slot
|
||
names to slot values. Only slots that have a value are
|
||
included in the latter.</p></li>
|
||
<li><p>For a class that has <a class="reference internal" href="../reference/datamodel.html#object.__slots__" title="object.__slots__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__slots__</span></code></a> and no instance
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__dict__" title="object.__dict__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__dict__</span></code></a>, the default state is a tuple whose first item
|
||
is <code class="docutils literal notranslate"><span class="pre">None</span></code> and whose second item is a dictionary mapping slot names
|
||
to slot values described in the previous bullet.</p></li>
|
||
</ul>
|
||
<div class="versionchanged">
|
||
<p><span class="versionmodified changed">Changed in version 3.11: </span>Added the default implementation of the <code class="docutils literal notranslate"><span class="pre">__getstate__()</span></code> method in the
|
||
<a class="reference internal" href="functions.html#object" title="object"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a> class.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__setstate__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__setstate__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#object.__setstate__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Upon unpickling, if the class defines <a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a>, it is called with
|
||
the unpickled state. In that case, there is no requirement for the state
|
||
object to be a dictionary. Otherwise, the pickled state must be a dictionary
|
||
and its items are assigned to the new instance’s dictionary.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>If <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> returns a state with value <code class="docutils literal notranslate"><span class="pre">None</span></code> at pickling,
|
||
the <a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a> method will not be called upon unpickling.</p>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<p>Refer to the section <a class="reference internal" href="#pickle-state"><span class="std std-ref">Handling Stateful Objects</span></a> for more information about how to use
|
||
the methods <a class="reference internal" href="#object.__getstate__" title="object.__getstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getstate__()</span></code></a> and <a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a>.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>At unpickling time, some methods like <a class="reference internal" href="../reference/datamodel.html#object.__getattr__" title="object.__getattr__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getattr__()</span></code></a>,
|
||
<a class="reference internal" href="../reference/datamodel.html#object.__getattribute__" title="object.__getattribute__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getattribute__()</span></code></a>, or <a class="reference internal" href="../reference/datamodel.html#object.__setattr__" title="object.__setattr__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setattr__()</span></code></a> may be called upon the
|
||
instance. In case those methods rely on some internal invariant being
|
||
true, the type should implement <a class="reference internal" href="../reference/datamodel.html#object.__new__" title="object.__new__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__new__()</span></code></a> to establish such an
|
||
invariant, as <a class="reference internal" href="../reference/datamodel.html#object.__init__" title="object.__init__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__init__()</span></code></a> is not called when unpickling an
|
||
instance.</p>
|
||
</div>
|
||
<p id="index-5">As we shall see, pickle does not use directly the methods described above. In
|
||
fact, these methods are part of the copy protocol which implements the
|
||
<a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> special method. The copy protocol provides a unified
|
||
interface for retrieving the data necessary for pickling and copying
|
||
objects. <a class="footnote-reference brackets" href="#id10" id="id5" role="doc-noteref"><span class="fn-bracket">[</span>4<span class="fn-bracket">]</span></a></p>
|
||
<p>Although powerful, implementing <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> directly in your classes is
|
||
error prone. For this reason, class designers should use the high-level
|
||
interface (i.e., <a class="reference internal" href="#object.__getnewargs_ex__" title="object.__getnewargs_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getnewargs_ex__()</span></code></a>, <a class="reference internal" href="#object.__getstate__" title="object.__getstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__getstate__()</span></code></a> and
|
||
<a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a>) whenever possible. We will show, however, cases where
|
||
using <code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code> is the only option or leads to more efficient pickling
|
||
or both.</p>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__reduce__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__reduce__</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#object.__reduce__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>The interface is currently defined as follows. The <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> method
|
||
takes no argument and shall return either a string or preferably a tuple (the
|
||
returned object is often referred to as the “reduce value”).</p>
|
||
<p>If a string is returned, the string should be interpreted as the name of a
|
||
global variable. It should be the object’s local name relative to its
|
||
module; the pickle module searches the module namespace to determine the
|
||
object’s module. This behaviour is typically useful for singletons.</p>
|
||
<p>When a tuple is returned, it must be between two and six items long.
|
||
Optional items can either be omitted, or <code class="docutils literal notranslate"><span class="pre">None</span></code> can be provided as their
|
||
value. The semantics of each item are in order:</p>
|
||
<ul>
|
||
<li><p>A callable object that will be called to create the initial version of the
|
||
object.</p></li>
|
||
<li><p>A tuple of arguments for the callable object. An empty tuple must be given
|
||
if the callable does not accept any argument.</p></li>
|
||
<li><p>Optionally, the object’s state, which will be passed to the object’s
|
||
<a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a> method as previously described. If the object has no
|
||
such method then, the value must be a dictionary and it will be added to
|
||
the object’s <a class="reference internal" href="../reference/datamodel.html#object.__dict__" title="object.__dict__"><code class="xref py py-attr docutils literal notranslate"><span class="pre">__dict__</span></code></a> attribute.</p></li>
|
||
<li><p>Optionally, an iterator (and not a sequence) yielding successive items.
|
||
These items will be appended to the object either using
|
||
<code class="docutils literal notranslate"><span class="pre">obj.append(item)</span></code> or, in batch, using <code class="docutils literal notranslate"><span class="pre">obj.extend(list_of_items)</span></code>.
|
||
This is primarily used for list subclasses, but may be used by other
|
||
classes as long as they have
|
||
<a class="reference internal" href="stdtypes.html#typesseq-common"><span class="std std-ref">append and extend methods</span></a> with
|
||
the appropriate signature. (Whether <code class="xref py py-meth docutils literal notranslate"><span class="pre">append()</span></code> or <code class="xref py py-meth docutils literal notranslate"><span class="pre">extend()</span></code> is
|
||
used depends on which pickle protocol version is used as well as the number
|
||
of items to append, so both must be supported.)</p></li>
|
||
<li><p>Optionally, an iterator (not a sequence) yielding successive key-value
|
||
pairs. These items will be stored to the object using <code class="docutils literal notranslate"><span class="pre">obj[key]</span> <span class="pre">=</span>
|
||
<span class="pre">value</span></code>. This is primarily used for dictionary subclasses, but may be used
|
||
by other classes as long as they implement <a class="reference internal" href="../reference/datamodel.html#object.__setitem__" title="object.__setitem__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setitem__()</span></code></a>.</p></li>
|
||
<li><p>Optionally, a callable with a <code class="docutils literal notranslate"><span class="pre">(obj,</span> <span class="pre">state)</span></code> signature. This
|
||
callable allows the user to programmatically control the state-updating
|
||
behavior of a specific object, instead of using <code class="docutils literal notranslate"><span class="pre">obj</span></code>’s static
|
||
<a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a> method. If not <code class="docutils literal notranslate"><span class="pre">None</span></code>, this callable will have
|
||
priority over <code class="docutils literal notranslate"><span class="pre">obj</span></code>’s <a class="reference internal" href="#object.__setstate__" title="object.__setstate__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code></a>.</p>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.8: </span>The optional sixth tuple item, <code class="docutils literal notranslate"><span class="pre">(obj,</span> <span class="pre">state)</span></code>, was added.</p>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="object.__reduce_ex__">
|
||
<span class="sig-prename descclassname"><span class="pre">object.</span></span><span class="sig-name descname"><span class="pre">__reduce_ex__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">protocol</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#object.__reduce_ex__" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Alternatively, a <a class="reference internal" href="#object.__reduce_ex__" title="object.__reduce_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce_ex__()</span></code></a> method may be defined. The only
|
||
difference is this method should take a single integer argument, the protocol
|
||
version. When defined, pickle will prefer it over the <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a>
|
||
method. In addition, <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a> automatically becomes a synonym for
|
||
the extended version. The main use for this method is to provide
|
||
backwards-compatible reduce values for older Python releases.</p>
|
||
</dd></dl>
|
||
|
||
<section id="persistence-of-external-objects">
|
||
<span id="pickle-persistent"></span><h3>Persistence of External Objects<a class="headerlink" href="#persistence-of-external-objects" title="Link to this heading">¶</a></h3>
|
||
<p id="index-6">For the benefit of object persistence, the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module supports the
|
||
notion of a reference to an object outside the pickled data stream. Such
|
||
objects are referenced by a persistent ID, which should be either a string of
|
||
alphanumeric characters (for protocol 0) <a class="footnote-reference brackets" href="#id11" id="id6" role="doc-noteref"><span class="fn-bracket">[</span>5<span class="fn-bracket">]</span></a> or just an arbitrary object (for
|
||
any newer protocol).</p>
|
||
<p>The resolution of such persistent IDs is not defined by the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a>
|
||
module; it will delegate this resolution to the user-defined methods on the
|
||
pickler and unpickler, <a class="reference internal" href="#pickle.Pickler.persistent_id" title="pickle.Pickler.persistent_id"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_id()</span></code></a> and
|
||
<a class="reference internal" href="#pickle.Unpickler.persistent_load" title="pickle.Unpickler.persistent_load"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_load()</span></code></a> respectively.</p>
|
||
<p>To pickle objects that have an external persistent ID, the pickler must have a
|
||
custom <a class="reference internal" href="#pickle.Pickler.persistent_id" title="pickle.Pickler.persistent_id"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_id()</span></code></a> method that takes an object as an
|
||
argument and returns either <code class="docutils literal notranslate"><span class="pre">None</span></code> or the persistent ID for that object.
|
||
When <code class="docutils literal notranslate"><span class="pre">None</span></code> is returned, the pickler simply pickles the object as normal.
|
||
When a persistent ID string is returned, the pickler will pickle that object,
|
||
along with a marker so that the unpickler will recognize it as a persistent ID.</p>
|
||
<p>To unpickle external objects, the unpickler must have a custom
|
||
<a class="reference internal" href="#pickle.Unpickler.persistent_load" title="pickle.Unpickler.persistent_load"><code class="xref py py-meth docutils literal notranslate"><span class="pre">persistent_load()</span></code></a> method that takes a persistent ID object and
|
||
returns the referenced object.</p>
|
||
<p>Here is a comprehensive example presenting how persistent ID can be used to
|
||
pickle external objects by reference.</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Simple example presenting how persistent ID can be used to pickle</span>
|
||
<span class="c1"># external objects by reference.</span>
|
||
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
|
||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">namedtuple</span>
|
||
|
||
<span class="c1"># Simple class representing a record in our database.</span>
|
||
<span class="n">MemoRecord</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="p">(</span><span class="s2">"MemoRecord"</span><span class="p">,</span> <span class="s2">"key, task"</span><span class="p">)</span>
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">DBPickler</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">Pickler</span><span class="p">):</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">persistent_id</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
|
||
<span class="c1"># Instead of pickling MemoRecord as a regular class instance, we emit a</span>
|
||
<span class="c1"># persistent ID.</span>
|
||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">MemoRecord</span><span class="p">):</span>
|
||
<span class="c1"># Here, our persistent ID is simply a tuple, containing a tag and a</span>
|
||
<span class="c1"># key, which refers to a specific record in the database.</span>
|
||
<span class="k">return</span> <span class="p">(</span><span class="s2">"MemoRecord"</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">key</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># If obj does not have a persistent ID, return None. This means obj</span>
|
||
<span class="c1"># needs to be pickled as usual.</span>
|
||
<span class="k">return</span> <span class="kc">None</span>
|
||
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">DBUnpickler</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">Unpickler</span><span class="p">):</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file</span><span class="p">,</span> <span class="n">connection</span><span class="p">):</span>
|
||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">connection</span> <span class="o">=</span> <span class="n">connection</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">persistent_load</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pid</span><span class="p">):</span>
|
||
<span class="c1"># This method is invoked whenever a persistent ID is encountered.</span>
|
||
<span class="c1"># Here, pid is the tuple returned by DBPickler.</span>
|
||
<span class="n">cursor</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">connection</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
|
||
<span class="n">type_tag</span><span class="p">,</span> <span class="n">key_id</span> <span class="o">=</span> <span class="n">pid</span>
|
||
<span class="k">if</span> <span class="n">type_tag</span> <span class="o">==</span> <span class="s2">"MemoRecord"</span><span class="p">:</span>
|
||
<span class="c1"># Fetch the referenced record from the database and return it.</span>
|
||
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"SELECT * FROM memos WHERE key=?"</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">key_id</span><span class="p">),))</span>
|
||
<span class="n">key</span><span class="p">,</span> <span class="n">task</span> <span class="o">=</span> <span class="n">cursor</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
|
||
<span class="k">return</span> <span class="n">MemoRecord</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">task</span><span class="p">)</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># Always raises an error if you cannot return the correct object.</span>
|
||
<span class="c1"># Otherwise, the unpickler will think None is the object referenced</span>
|
||
<span class="c1"># by the persistent ID.</span>
|
||
<span class="k">raise</span> <span class="n">pickle</span><span class="o">.</span><span class="n">UnpicklingError</span><span class="p">(</span><span class="s2">"unsupported persistent object"</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">main</span><span class="p">():</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">pprint</span>
|
||
|
||
<span class="c1"># Initialize and populate our database.</span>
|
||
<span class="n">conn</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="s2">":memory:"</span><span class="p">)</span>
|
||
<span class="n">cursor</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">cursor</span><span class="p">()</span>
|
||
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"CREATE TABLE memos(key INTEGER PRIMARY KEY, task TEXT)"</span><span class="p">)</span>
|
||
<span class="n">tasks</span> <span class="o">=</span> <span class="p">(</span>
|
||
<span class="s1">'give food to fish'</span><span class="p">,</span>
|
||
<span class="s1">'prepare group meeting'</span><span class="p">,</span>
|
||
<span class="s1">'fight with a zebra'</span><span class="p">,</span>
|
||
<span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">tasks</span><span class="p">:</span>
|
||
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"INSERT INTO memos VALUES(NULL, ?)"</span><span class="p">,</span> <span class="p">(</span><span class="n">task</span><span class="p">,))</span>
|
||
|
||
<span class="c1"># Fetch the records to be pickled.</span>
|
||
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"SELECT * FROM memos"</span><span class="p">)</span>
|
||
<span class="n">memos</span> <span class="o">=</span> <span class="p">[</span><span class="n">MemoRecord</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">task</span><span class="p">)</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">task</span> <span class="ow">in</span> <span class="n">cursor</span><span class="p">]</span>
|
||
<span class="c1"># Save the records using our custom DBPickler.</span>
|
||
<span class="n">file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
|
||
<span class="n">DBPickler</span><span class="p">(</span><span class="n">file</span><span class="p">)</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">memos</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Pickled records:"</span><span class="p">)</span>
|
||
<span class="n">pprint</span><span class="o">.</span><span class="n">pprint</span><span class="p">(</span><span class="n">memos</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Update a record, just for good measure.</span>
|
||
<span class="n">cursor</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"UPDATE memos SET task='learn italian' WHERE key=1"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Load the records from the pickle data stream.</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">memos</span> <span class="o">=</span> <span class="n">DBUnpickler</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="n">conn</span><span class="p">)</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Unpickled records:"</span><span class="p">)</span>
|
||
<span class="n">pprint</span><span class="o">.</span><span class="n">pprint</span><span class="p">(</span><span class="n">memos</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||
<span class="n">main</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="dispatch-tables">
|
||
<span id="pickle-dispatch"></span><h3>Dispatch Tables<a class="headerlink" href="#dispatch-tables" title="Link to this heading">¶</a></h3>
|
||
<p>If one wants to customize pickling of some classes without disturbing
|
||
any other code which depends on pickling, then one can create a
|
||
pickler with a private dispatch table.</p>
|
||
<p>The global dispatch table managed by the <a class="reference internal" href="copyreg.html#module-copyreg" title="copyreg: Register pickle support functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code></a> module is
|
||
available as <code class="xref py py-data docutils literal notranslate"><span class="pre">copyreg.dispatch_table</span></code>. Therefore, one may
|
||
choose to use a modified copy of <code class="xref py py-data docutils literal notranslate"><span class="pre">copyreg.dispatch_table</span></code> as a
|
||
private dispatch table.</p>
|
||
<p>For example</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="n">f</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">Pickler</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
<span class="n">p</span><span class="o">.</span><span class="n">dispatch_table</span> <span class="o">=</span> <span class="n">copyreg</span><span class="o">.</span><span class="n">dispatch_table</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="n">p</span><span class="o">.</span><span class="n">dispatch_table</span><span class="p">[</span><span class="n">SomeClass</span><span class="p">]</span> <span class="o">=</span> <span class="n">reduce_SomeClass</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>creates an instance of <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">pickle.Pickler</span></code></a> with a private dispatch
|
||
table which handles the <code class="docutils literal notranslate"><span class="pre">SomeClass</span></code> class specially. Alternatively,
|
||
the code</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">MyPickler</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">Pickler</span><span class="p">):</span>
|
||
<span class="n">dispatch_table</span> <span class="o">=</span> <span class="n">copyreg</span><span class="o">.</span><span class="n">dispatch_table</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="n">dispatch_table</span><span class="p">[</span><span class="n">SomeClass</span><span class="p">]</span> <span class="o">=</span> <span class="n">reduce_SomeClass</span>
|
||
<span class="n">f</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">MyPickler</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>does the same but all instances of <code class="docutils literal notranslate"><span class="pre">MyPickler</span></code> will by default
|
||
share the private dispatch table. On the other hand, the code</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="n">copyreg</span><span class="o">.</span><span class="n">pickle</span><span class="p">(</span><span class="n">SomeClass</span><span class="p">,</span> <span class="n">reduce_SomeClass</span><span class="p">)</span>
|
||
<span class="n">f</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">Pickler</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>modifies the global dispatch table shared by all users of the <a class="reference internal" href="copyreg.html#module-copyreg" title="copyreg: Register pickle support functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code></a> module.</p>
|
||
</section>
|
||
<section id="handling-stateful-objects">
|
||
<span id="pickle-state"></span><h3>Handling Stateful Objects<a class="headerlink" href="#handling-stateful-objects" title="Link to this heading">¶</a></h3>
|
||
<p id="index-7">Here’s an example that shows how to modify pickling behavior for a class.
|
||
The <code class="xref py py-class docutils literal notranslate"><span class="pre">TextReader</span></code> class below opens a text file, and returns the line number and
|
||
line contents each time its <code class="xref py py-meth docutils literal notranslate"><span class="pre">readline()</span></code> method is called. If a
|
||
<code class="xref py py-class docutils literal notranslate"><span class="pre">TextReader</span></code> instance is pickled, all attributes <em>except</em> the file object
|
||
member are saved. When the instance is unpickled, the file is reopened, and
|
||
reading resumes from the last location. The <code class="xref py py-meth docutils literal notranslate"><span class="pre">__setstate__()</span></code> and
|
||
<code class="xref py py-meth docutils literal notranslate"><span class="pre">__getstate__()</span></code> methods are used to implement this behavior.</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">TextReader</span><span class="p">:</span>
|
||
<span class="w"> </span><span class="sd">"""Print and number lines in a text file."""</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">lineno</span> <span class="o">=</span> <span class="mi">0</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">readline</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">lineno</span> <span class="o">+=</span> <span class="mi">1</span>
|
||
<span class="n">line</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">file</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">line</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="kc">None</span>
|
||
<span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">):</span>
|
||
<span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="k">return</span> <span class="s2">"</span><span class="si">%i</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lineno</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="c1"># Copy the object's state from self.__dict__ which contains</span>
|
||
<span class="c1"># all our instance attributes. Always use the dict.copy()</span>
|
||
<span class="c1"># method to avoid modifying the original state.</span>
|
||
<span class="n">state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="c1"># Remove the unpicklable entries.</span>
|
||
<span class="k">del</span> <span class="n">state</span><span class="p">[</span><span class="s1">'file'</span><span class="p">]</span>
|
||
<span class="k">return</span> <span class="n">state</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
|
||
<span class="c1"># Restore instance attributes (i.e., filename and lineno).</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>
|
||
<span class="c1"># Restore the previously opened file's state. To do so, we need to</span>
|
||
<span class="c1"># reopen it and read from it until the line count is restored.</span>
|
||
<span class="n">file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lineno</span><span class="p">):</span>
|
||
<span class="n">file</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||
<span class="c1"># Finally, save the file.</span>
|
||
<span class="bp">self</span><span class="o">.</span><span class="n">file</span> <span class="o">=</span> <span class="n">file</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>A sample usage might be something like this:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">reader</span> <span class="o">=</span> <span class="n">TextReader</span><span class="p">(</span><span class="s2">"hello.txt"</span><span class="p">)</span>
|
||
<span class="gp">>>> </span><span class="n">reader</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||
<span class="go">'1: Hello world!'</span>
|
||
<span class="gp">>>> </span><span class="n">reader</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||
<span class="go">'2: I am line number two.'</span>
|
||
<span class="gp">>>> </span><span class="n">new_reader</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">reader</span><span class="p">))</span>
|
||
<span class="gp">>>> </span><span class="n">new_reader</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||
<span class="go">'3: Goodbye!'</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="custom-reduction-for-types-functions-and-other-objects">
|
||
<span id="reducer-override"></span><h2>Custom Reduction for Types, Functions, and Other Objects<a class="headerlink" href="#custom-reduction-for-types-functions-and-other-objects" title="Link to this heading">¶</a></h2>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.8.</span></p>
|
||
</div>
|
||
<p>Sometimes, <a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a> may not be flexible enough.
|
||
In particular we may want to customize pickling based on another criterion
|
||
than the object’s type, or we may want to customize the pickling of
|
||
functions and classes.</p>
|
||
<p>For those cases, it is possible to subclass from the <a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> class and
|
||
implement a <a class="reference internal" href="#pickle.Pickler.reducer_override" title="pickle.Pickler.reducer_override"><code class="xref py py-meth docutils literal notranslate"><span class="pre">reducer_override()</span></code></a> method. This method can return an
|
||
arbitrary reduction tuple (see <a class="reference internal" href="#object.__reduce__" title="object.__reduce__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce__()</span></code></a>). It can alternatively return
|
||
<a class="reference internal" href="constants.html#NotImplemented" title="NotImplemented"><code class="xref py py-data docutils literal notranslate"><span class="pre">NotImplemented</span></code></a> to fallback to the traditional behavior.</p>
|
||
<p>If both the <a class="reference internal" href="#pickle.Pickler.dispatch_table" title="pickle.Pickler.dispatch_table"><code class="xref py py-attr docutils literal notranslate"><span class="pre">dispatch_table</span></code></a> and
|
||
<a class="reference internal" href="#pickle.Pickler.reducer_override" title="pickle.Pickler.reducer_override"><code class="xref py py-meth docutils literal notranslate"><span class="pre">reducer_override()</span></code></a> are defined, then
|
||
<a class="reference internal" href="#pickle.Pickler.reducer_override" title="pickle.Pickler.reducer_override"><code class="xref py py-meth docutils literal notranslate"><span class="pre">reducer_override()</span></code></a> method takes priority.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>For performance reasons, <a class="reference internal" href="#pickle.Pickler.reducer_override" title="pickle.Pickler.reducer_override"><code class="xref py py-meth docutils literal notranslate"><span class="pre">reducer_override()</span></code></a> may not be
|
||
called for the following objects: <code class="docutils literal notranslate"><span class="pre">None</span></code>, <code class="docutils literal notranslate"><span class="pre">True</span></code>, <code class="docutils literal notranslate"><span class="pre">False</span></code>, and
|
||
exact instances of <a class="reference internal" href="functions.html#int" title="int"><code class="xref py py-class docutils literal notranslate"><span class="pre">int</span></code></a>, <a class="reference internal" href="functions.html#float" title="float"><code class="xref py py-class docutils literal notranslate"><span class="pre">float</span></code></a>, <a class="reference internal" href="stdtypes.html#bytes" title="bytes"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytes</span></code></a>,
|
||
<a class="reference internal" href="stdtypes.html#str" title="str"><code class="xref py py-class docutils literal notranslate"><span class="pre">str</span></code></a>, <a class="reference internal" href="stdtypes.html#dict" title="dict"><code class="xref py py-class docutils literal notranslate"><span class="pre">dict</span></code></a>, <a class="reference internal" href="stdtypes.html#set" title="set"><code class="xref py py-class docutils literal notranslate"><span class="pre">set</span></code></a>, <a class="reference internal" href="stdtypes.html#frozenset" title="frozenset"><code class="xref py py-class docutils literal notranslate"><span class="pre">frozenset</span></code></a>, <a class="reference internal" href="stdtypes.html#list" title="list"><code class="xref py py-class docutils literal notranslate"><span class="pre">list</span></code></a>
|
||
and <a class="reference internal" href="stdtypes.html#tuple" title="tuple"><code class="xref py py-class docutils literal notranslate"><span class="pre">tuple</span></code></a>.</p>
|
||
</div>
|
||
<p>Here is a simple example where we allow pickling and reconstructing
|
||
a given class:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">MyClass</span><span class="p">:</span>
|
||
<span class="n">my_attribute</span> <span class="o">=</span> <span class="mi">1</span>
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">MyPickler</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">Pickler</span><span class="p">):</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">reducer_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Custom reducer for MyClass."""</span>
|
||
<span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s2">"__name__"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="s2">"MyClass"</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="nb">type</span><span class="p">,</span> <span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__bases__</span><span class="p">,</span>
|
||
<span class="p">{</span><span class="s1">'my_attribute'</span><span class="p">:</span> <span class="n">obj</span><span class="o">.</span><span class="n">my_attribute</span><span class="p">})</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># For any other object, fallback to usual reduction</span>
|
||
<span class="k">return</span> <span class="bp">NotImplemented</span>
|
||
|
||
<span class="n">f</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">()</span>
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">MyPickler</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
<span class="n">p</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">MyClass</span><span class="p">)</span>
|
||
|
||
<span class="k">del</span> <span class="n">MyClass</span>
|
||
|
||
<span class="n">unpickled_class</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">getvalue</span><span class="p">())</span>
|
||
|
||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">unpickled_class</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
|
||
<span class="k">assert</span> <span class="n">unpickled_class</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">"MyClass"</span>
|
||
<span class="k">assert</span> <span class="n">unpickled_class</span><span class="o">.</span><span class="n">my_attribute</span> <span class="o">==</span> <span class="mi">1</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="out-of-band-buffers">
|
||
<span id="pickle-oob"></span><h2>Out-of-band Buffers<a class="headerlink" href="#out-of-band-buffers" title="Link to this heading">¶</a></h2>
|
||
<div class="versionadded">
|
||
<p><span class="versionmodified added">Added in version 3.8.</span></p>
|
||
</div>
|
||
<p>In some contexts, the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module is used to transfer massive amounts
|
||
of data. Therefore, it can be important to minimize the number of memory
|
||
copies, to preserve performance and resource consumption. However, normal
|
||
operation of the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module, as it transforms a graph-like structure
|
||
of objects into a sequential stream of bytes, intrinsically involves copying
|
||
data to and from the pickle stream.</p>
|
||
<p>This constraint can be eschewed if both the <em>provider</em> (the implementation
|
||
of the object types to be transferred) and the <em>consumer</em> (the implementation
|
||
of the communications system) support the out-of-band transfer facilities
|
||
provided by pickle protocol 5 and higher.</p>
|
||
<section id="provider-api">
|
||
<h3>Provider API<a class="headerlink" href="#provider-api" title="Link to this heading">¶</a></h3>
|
||
<p>The large data objects to be pickled must implement a <a class="reference internal" href="#object.__reduce_ex__" title="object.__reduce_ex__"><code class="xref py py-meth docutils literal notranslate"><span class="pre">__reduce_ex__()</span></code></a>
|
||
method specialized for protocol 5 and higher, which returns a
|
||
<a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> instance (instead of e.g. a <a class="reference internal" href="stdtypes.html#bytes" title="bytes"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytes</span></code></a> object)
|
||
for any large data.</p>
|
||
<p>A <a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> object <em>signals</em> that the underlying buffer is
|
||
eligible for out-of-band data transfer. Those objects remain compatible
|
||
with normal usage of the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module. However, consumers can also
|
||
opt-in to tell <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> that they will handle those buffers by
|
||
themselves.</p>
|
||
</section>
|
||
<section id="consumer-api">
|
||
<h3>Consumer API<a class="headerlink" href="#consumer-api" title="Link to this heading">¶</a></h3>
|
||
<p>A communications system can enable custom handling of the <a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a>
|
||
objects generated when serializing an object graph.</p>
|
||
<p>On the sending side, it needs to pass a <em>buffer_callback</em> argument to
|
||
<a class="reference internal" href="#pickle.Pickler" title="pickle.Pickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Pickler</span></code></a> (or to the <a class="reference internal" href="#pickle.dump" title="pickle.dump"><code class="xref py py-func docutils literal notranslate"><span class="pre">dump()</span></code></a> or <a class="reference internal" href="#pickle.dumps" title="pickle.dumps"><code class="xref py py-func docutils literal notranslate"><span class="pre">dumps()</span></code></a> function), which
|
||
will be called with each <a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> generated while pickling
|
||
the object graph. Buffers accumulated by the <em>buffer_callback</em> will not
|
||
see their data copied into the pickle stream, only a cheap marker will be
|
||
inserted.</p>
|
||
<p>On the receiving side, it needs to pass a <em>buffers</em> argument to
|
||
<a class="reference internal" href="#pickle.Unpickler" title="pickle.Unpickler"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unpickler</span></code></a> (or to the <a class="reference internal" href="#pickle.load" title="pickle.load"><code class="xref py py-func docutils literal notranslate"><span class="pre">load()</span></code></a> or <a class="reference internal" href="#pickle.loads" title="pickle.loads"><code class="xref py py-func docutils literal notranslate"><span class="pre">loads()</span></code></a> function),
|
||
which is an iterable of the buffers which were passed to <em>buffer_callback</em>.
|
||
That iterable should produce buffers in the same order as they were passed
|
||
to <em>buffer_callback</em>. Those buffers will provide the data expected by the
|
||
reconstructors of the objects whose pickling produced the original
|
||
<a class="reference internal" href="#pickle.PickleBuffer" title="pickle.PickleBuffer"><code class="xref py py-class docutils literal notranslate"><span class="pre">PickleBuffer</span></code></a> objects.</p>
|
||
<p>Between the sending side and the receiving side, the communications system
|
||
is free to implement its own transfer mechanism for out-of-band buffers.
|
||
Potential optimizations include the use of shared memory or datatype-dependent
|
||
compression.</p>
|
||
</section>
|
||
<section id="example">
|
||
<h3>Example<a class="headerlink" href="#example" title="Link to this heading">¶</a></h3>
|
||
<p>Here is a trivial example where we implement a <a class="reference internal" href="stdtypes.html#bytearray" title="bytearray"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytearray</span></code></a> subclass
|
||
able to participate in out-of-band buffer pickling:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">ZeroCopyByteArray</span><span class="p">(</span><span class="nb">bytearray</span><span class="p">):</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">__reduce_ex__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">protocol</span><span class="p">):</span>
|
||
<span class="k">if</span> <span class="n">protocol</span> <span class="o">>=</span> <span class="mi">5</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_reconstruct</span><span class="p">,</span> <span class="p">(</span><span class="n">PickleBuffer</span><span class="p">(</span><span class="bp">self</span><span class="p">),),</span> <span class="kc">None</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="c1"># PickleBuffer is forbidden with pickle protocols <= 4.</span>
|
||
<span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_reconstruct</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytearray</span><span class="p">(</span><span class="bp">self</span><span class="p">),)</span>
|
||
|
||
<span class="nd">@classmethod</span>
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">_reconstruct</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
|
||
<span class="k">with</span> <span class="nb">memoryview</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="k">as</span> <span class="n">m</span><span class="p">:</span>
|
||
<span class="c1"># Get a handle over the original buffer object</span>
|
||
<span class="n">obj</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">obj</span>
|
||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="ow">is</span> <span class="bp">cls</span><span class="p">:</span>
|
||
<span class="c1"># Original buffer object is a ZeroCopyByteArray, return it</span>
|
||
<span class="c1"># as-is.</span>
|
||
<span class="k">return</span> <span class="n">obj</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The reconstructor (the <code class="docutils literal notranslate"><span class="pre">_reconstruct</span></code> class method) returns the buffer’s
|
||
providing object if it has the right type. This is an easy way to simulate
|
||
zero-copy behaviour on this toy example.</p>
|
||
<p>On the consumer side, we can pickle those objects the usual way, which
|
||
when unserialized will give us a copy of the original object:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="n">b</span> <span class="o">=</span> <span class="n">ZeroCopyByteArray</span><span class="p">(</span><span class="sa">b</span><span class="s2">"abc"</span><span class="p">)</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
|
||
<span class="n">new_b</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">b</span> <span class="o">==</span> <span class="n">new_b</span><span class="p">)</span> <span class="c1"># True</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">b</span> <span class="ow">is</span> <span class="n">new_b</span><span class="p">)</span> <span class="c1"># False: a copy was made</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>But if we pass a <em>buffer_callback</em> and then give back the accumulated
|
||
buffers when unserializing, we are able to get back the original object:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="n">b</span> <span class="o">=</span> <span class="n">ZeroCopyByteArray</span><span class="p">(</span><span class="sa">b</span><span class="s2">"abc"</span><span class="p">)</span>
|
||
<span class="n">buffers</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="n">protocol</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">buffer_callback</span><span class="o">=</span><span class="n">buffers</span><span class="o">.</span><span class="n">append</span><span class="p">)</span>
|
||
<span class="n">new_b</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">buffers</span><span class="o">=</span><span class="n">buffers</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">b</span> <span class="o">==</span> <span class="n">new_b</span><span class="p">)</span> <span class="c1"># True</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">b</span> <span class="ow">is</span> <span class="n">new_b</span><span class="p">)</span> <span class="c1"># True: no copy was made</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>This example is limited by the fact that <a class="reference internal" href="stdtypes.html#bytearray" title="bytearray"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytearray</span></code></a> allocates its
|
||
own memory: you cannot create a <a class="reference internal" href="stdtypes.html#bytearray" title="bytearray"><code class="xref py py-class docutils literal notranslate"><span class="pre">bytearray</span></code></a> instance that is backed
|
||
by another object’s memory. However, third-party datatypes such as NumPy
|
||
arrays do not have this limitation, and allow use of zero-copy pickling
|
||
(or making as few copies as possible) when transferring between distinct
|
||
processes or systems.</p>
|
||
<div class="admonition seealso">
|
||
<p class="admonition-title">See also</p>
|
||
<p><span class="target" id="index-8"></span><a class="pep reference external" href="https://peps.python.org/pep-0574/"><strong>PEP 574</strong></a> – Pickle protocol 5 with out-of-band data</p>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="restricting-globals">
|
||
<span id="pickle-restrict"></span><h2>Restricting Globals<a class="headerlink" href="#restricting-globals" title="Link to this heading">¶</a></h2>
|
||
<p id="index-9">By default, unpickling will import any class or function that it finds in the
|
||
pickle data. For many applications, this behaviour is unacceptable as it
|
||
permits the unpickler to import and invoke arbitrary code. Just consider what
|
||
this hand-crafted pickle data stream does when loaded:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
<span class="gp">>>> </span><span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="sa">b</span><span class="s2">"cos</span><span class="se">\n</span><span class="s2">system</span><span class="se">\n</span><span class="s2">(S'echo hello world'</span><span class="se">\n</span><span class="s2">tR."</span><span class="p">)</span>
|
||
<span class="go">hello world</span>
|
||
<span class="go">0</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>In this example, the unpickler imports the <a class="reference internal" href="os.html#os.system" title="os.system"><code class="xref py py-func docutils literal notranslate"><span class="pre">os.system()</span></code></a> function and then
|
||
apply the string argument “echo hello world”. Although this example is
|
||
inoffensive, it is not difficult to imagine one that could damage your system.</p>
|
||
<p>For this reason, you may want to control what gets unpickled by customizing
|
||
<a class="reference internal" href="#pickle.Unpickler.find_class" title="pickle.Unpickler.find_class"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Unpickler.find_class()</span></code></a>. Unlike its name suggests,
|
||
<a class="reference internal" href="#pickle.Unpickler.find_class" title="pickle.Unpickler.find_class"><code class="xref py py-meth docutils literal notranslate"><span class="pre">Unpickler.find_class()</span></code></a> is called whenever a global (i.e., a class or
|
||
a function) is requested. Thus it is possible to either completely forbid
|
||
globals or restrict them to a safe subset.</p>
|
||
<p>Here is an example of an unpickler allowing only few safe classes from the
|
||
<a class="reference internal" href="builtins.html#module-builtins" title="builtins: The module that provides the built-in namespace."><code class="xref py py-mod docutils literal notranslate"><span class="pre">builtins</span></code></a> module to be loaded:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">builtins</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
|
||
<span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
|
||
<span class="n">safe_builtins</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'range'</span><span class="p">,</span>
|
||
<span class="s1">'complex'</span><span class="p">,</span>
|
||
<span class="s1">'set'</span><span class="p">,</span>
|
||
<span class="s1">'frozenset'</span><span class="p">,</span>
|
||
<span class="s1">'slice'</span><span class="p">,</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="k">class</span><span class="w"> </span><span class="nc">RestrictedUnpickler</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">Unpickler</span><span class="p">):</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">find_class</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||
<span class="c1"># Only allow safe classes from builtins.</span>
|
||
<span class="k">if</span> <span class="n">module</span> <span class="o">==</span> <span class="s2">"builtins"</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">safe_builtins</span><span class="p">:</span>
|
||
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">builtins</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
|
||
<span class="c1"># Forbid everything else.</span>
|
||
<span class="k">raise</span> <span class="n">pickle</span><span class="o">.</span><span class="n">UnpicklingError</span><span class="p">(</span><span class="s2">"global '</span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">' is forbidden"</span> <span class="o">%</span>
|
||
<span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>
|
||
|
||
<span class="k">def</span><span class="w"> </span><span class="nf">restricted_loads</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">"""Helper function analogous to pickle.loads()."""</span>
|
||
<span class="k">return</span> <span class="n">RestrictedUnpickler</span><span class="p">(</span><span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">s</span><span class="p">))</span><span class="o">.</span><span class="n">load</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>A sample usage of our unpickler working as intended:</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">restricted_loads</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="nb">range</span><span class="p">(</span><span class="mi">15</span><span class="p">)]))</span>
|
||
<span class="go">[1, 2, range(0, 15)]</span>
|
||
<span class="gp">>>> </span><span class="n">restricted_loads</span><span class="p">(</span><span class="sa">b</span><span class="s2">"cos</span><span class="se">\n</span><span class="s2">system</span><span class="se">\n</span><span class="s2">(S'echo hello world'</span><span class="se">\n</span><span class="s2">tR."</span><span class="p">)</span>
|
||
<span class="gt">Traceback (most recent call last):</span>
|
||
<span class="w"> </span><span class="c">...</span>
|
||
<span class="gr">pickle.UnpicklingError</span>: <span class="n">global 'os.system' is forbidden</span>
|
||
<span class="gp">>>> </span><span class="n">restricted_loads</span><span class="p">(</span><span class="sa">b</span><span class="s1">'cbuiltins</span><span class="se">\n</span><span class="s1">eval</span><span class="se">\n</span><span class="s1">'</span>
|
||
<span class="gp">... </span> <span class="sa">b</span><span class="s1">'(S</span><span class="se">\'</span><span class="s1">getattr(__import__("os"), "system")'</span>
|
||
<span class="gp">... </span> <span class="sa">b</span><span class="s1">'("echo hello world")</span><span class="se">\'\n</span><span class="s1">tR.'</span><span class="p">)</span>
|
||
<span class="gt">Traceback (most recent call last):</span>
|
||
<span class="w"> </span><span class="c">...</span>
|
||
<span class="gr">pickle.UnpicklingError</span>: <span class="n">global 'builtins.eval' is forbidden</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>As our examples shows, you have to be careful with what you allow to be
|
||
unpickled. Therefore if security is a concern, you may want to consider
|
||
alternatives such as the marshalling API in <a class="reference internal" href="xmlrpc.client.html#module-xmlrpc.client" title="xmlrpc.client: XML-RPC client access."><code class="xref py py-mod docutils literal notranslate"><span class="pre">xmlrpc.client</span></code></a> or
|
||
third-party solutions.</p>
|
||
</section>
|
||
<section id="performance">
|
||
<h2>Performance<a class="headerlink" href="#performance" title="Link to this heading">¶</a></h2>
|
||
<p>Recent versions of the pickle protocol (from protocol 2 and upwards) feature
|
||
efficient binary encodings for several common features and built-in types.
|
||
Also, the <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a> module has a transparent optimizer written in C.</p>
|
||
</section>
|
||
<section id="examples">
|
||
<span id="pickle-example"></span><h2>Examples<a class="headerlink" href="#examples" title="Link to this heading">¶</a></h2>
|
||
<p>For the simplest code, use the <a class="reference internal" href="#pickle.dump" title="pickle.dump"><code class="xref py py-func docutils literal notranslate"><span class="pre">dump()</span></code></a> and <a class="reference internal" href="#pickle.load" title="pickle.load"><code class="xref py py-func docutils literal notranslate"><span class="pre">load()</span></code></a> functions.</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
|
||
<span class="c1"># An arbitrary collection of objects supported by pickle.</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="p">{</span>
|
||
<span class="s1">'a'</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="mi">3</span><span class="o">+</span><span class="mi">4</span><span class="n">j</span><span class="p">],</span>
|
||
<span class="s1">'b'</span><span class="p">:</span> <span class="p">(</span><span class="s2">"character string"</span><span class="p">,</span> <span class="sa">b</span><span class="s2">"byte string"</span><span class="p">),</span>
|
||
<span class="s1">'c'</span><span class="p">:</span> <span class="p">{</span><span class="kc">None</span><span class="p">,</span> <span class="kc">True</span><span class="p">,</span> <span class="kc">False</span><span class="p">}</span>
|
||
<span class="p">}</span>
|
||
|
||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">'data.pickle'</span><span class="p">,</span> <span class="s1">'wb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||
<span class="c1"># Pickle the 'data' dictionary using the highest protocol available.</span>
|
||
<span class="n">pickle</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">pickle</span><span class="o">.</span><span class="n">HIGHEST_PROTOCOL</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>The following example reads the resulting pickled data.</p>
|
||
<div class="highlight-python3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||
|
||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s1">'data.pickle'</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||
<span class="c1"># The protocol version used is detected automatically, so we do not</span>
|
||
<span class="c1"># have to specify it.</span>
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<div class="admonition seealso">
|
||
<p class="admonition-title">See also</p>
|
||
<dl class="simple">
|
||
<dt>Module <a class="reference internal" href="copyreg.html#module-copyreg" title="copyreg: Register pickle support functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code></a></dt><dd><p>Pickle interface constructor registration for extension types.</p>
|
||
</dd>
|
||
<dt>Module <a class="reference internal" href="pickletools.html#module-pickletools" title="pickletools: Contains extensive comments about the pickle protocols and pickle-machine opcodes, as well as some useful functions."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickletools</span></code></a></dt><dd><p>Tools for working with and analyzing pickled data.</p>
|
||
</dd>
|
||
<dt>Module <a class="reference internal" href="shelve.html#module-shelve" title="shelve: Python object persistence."><code class="xref py py-mod docutils literal notranslate"><span class="pre">shelve</span></code></a></dt><dd><p>Indexed databases of objects; uses <a class="reference internal" href="#module-pickle" title="pickle: Convert Python objects to streams of bytes and back."><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code></a>.</p>
|
||
</dd>
|
||
<dt>Module <a class="reference internal" href="copy.html#module-copy" title="copy: Shallow and deep copy operations."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copy</span></code></a></dt><dd><p>Shallow and deep object copying.</p>
|
||
</dd>
|
||
<dt>Module <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a></dt><dd><p>High-performance serialization of built-in types.</p>
|
||
</dd>
|
||
</dl>
|
||
</div>
|
||
<p class="rubric">Footnotes</p>
|
||
<aside class="footnote-list brackets">
|
||
<aside class="footnote brackets" id="id7" role="doc-footnote">
|
||
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id1">1</a><span class="fn-bracket">]</span></span>
|
||
<p>Don’t confuse this with the <a class="reference internal" href="marshal.html#module-marshal" title="marshal: Convert Python objects to streams of bytes and back (with different constraints)."><code class="xref py py-mod docutils literal notranslate"><span class="pre">marshal</span></code></a> module</p>
|
||
</aside>
|
||
<aside class="footnote brackets" id="id8" role="doc-footnote">
|
||
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id3">2</a><span class="fn-bracket">]</span></span>
|
||
<p>This is why <a class="reference internal" href="../reference/expressions.html#lambda"><code class="xref std std-keyword docutils literal notranslate"><span class="pre">lambda</span></code></a> functions cannot be pickled: all
|
||
<code class="xref std std-keyword docutils literal notranslate"><span class="pre">lambda</span></code> functions share the same name: <code class="docutils literal notranslate"><span class="pre"><lambda></span></code>.</p>
|
||
</aside>
|
||
<aside class="footnote brackets" id="id9" role="doc-footnote">
|
||
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id4">3</a><span class="fn-bracket">]</span></span>
|
||
<p>The exception raised will likely be an <a class="reference internal" href="exceptions.html#ImportError" title="ImportError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">ImportError</span></code></a> or an
|
||
<a class="reference internal" href="exceptions.html#AttributeError" title="AttributeError"><code class="xref py py-exc docutils literal notranslate"><span class="pre">AttributeError</span></code></a> but it could be something else.</p>
|
||
</aside>
|
||
<aside class="footnote brackets" id="id10" role="doc-footnote">
|
||
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id5">4</a><span class="fn-bracket">]</span></span>
|
||
<p>The <a class="reference internal" href="copy.html#module-copy" title="copy: Shallow and deep copy operations."><code class="xref py py-mod docutils literal notranslate"><span class="pre">copy</span></code></a> module uses this protocol for shallow and deep copying
|
||
operations.</p>
|
||
</aside>
|
||
<aside class="footnote brackets" id="id11" role="doc-footnote">
|
||
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id6">5</a><span class="fn-bracket">]</span></span>
|
||
<p>The limitation on alphanumeric characters is due to the fact
|
||
that persistent IDs in protocol 0 are delimited by the newline
|
||
character. Therefore if any kind of newline characters occurs in
|
||
persistent IDs, the resulting pickled data will become unreadable.</p>
|
||
</aside>
|
||
</aside>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
<div class="clearer"></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||
<div class="sphinxsidebarwrapper">
|
||
<div>
|
||
<h3><a href="../contents.html">Table of Contents</a></h3>
|
||
<ul>
|
||
<li><a class="reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> — Python object serialization</a><ul>
|
||
<li><a class="reference internal" href="#relationship-to-other-python-modules">Relationship to other Python modules</a><ul>
|
||
<li><a class="reference internal" href="#comparison-with-marshal">Comparison with <code class="docutils literal notranslate"><span class="pre">marshal</span></code></a></li>
|
||
<li><a class="reference internal" href="#comparison-with-json">Comparison with <code class="docutils literal notranslate"><span class="pre">json</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#data-stream-format">Data stream format</a></li>
|
||
<li><a class="reference internal" href="#module-interface">Module Interface</a></li>
|
||
<li><a class="reference internal" href="#what-can-be-pickled-and-unpickled">What can be pickled and unpickled?</a></li>
|
||
<li><a class="reference internal" href="#pickling-class-instances">Pickling Class Instances</a><ul>
|
||
<li><a class="reference internal" href="#persistence-of-external-objects">Persistence of External Objects</a></li>
|
||
<li><a class="reference internal" href="#dispatch-tables">Dispatch Tables</a></li>
|
||
<li><a class="reference internal" href="#handling-stateful-objects">Handling Stateful Objects</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#custom-reduction-for-types-functions-and-other-objects">Custom Reduction for Types, Functions, and Other Objects</a></li>
|
||
<li><a class="reference internal" href="#out-of-band-buffers">Out-of-band Buffers</a><ul>
|
||
<li><a class="reference internal" href="#provider-api">Provider API</a></li>
|
||
<li><a class="reference internal" href="#consumer-api">Consumer API</a></li>
|
||
<li><a class="reference internal" href="#example">Example</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#restricting-globals">Restricting Globals</a></li>
|
||
<li><a class="reference internal" href="#performance">Performance</a></li>
|
||
<li><a class="reference internal" href="#examples">Examples</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
<div>
|
||
<h4>Previous topic</h4>
|
||
<p class="topless"><a href="persistence.html"
|
||
title="previous chapter">Data Persistence</a></p>
|
||
</div>
|
||
<div>
|
||
<h4>Next topic</h4>
|
||
<p class="topless"><a href="copyreg.html"
|
||
title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">copyreg</span></code> — Register <code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> support functions</a></p>
|
||
</div>
|
||
<div role="note" aria-label="source link">
|
||
<h3>This Page</h3>
|
||
<ul class="this-page-menu">
|
||
<li><a href="../bugs.html">Report a Bug</a></li>
|
||
<li>
|
||
<a href="https://github.com/python/cpython/blob/main/Doc/library/pickle.rst"
|
||
rel="nofollow">Show Source
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
<div id="sidebarbutton" title="Collapse sidebar">
|
||
<span>«</span>
|
||
</div>
|
||
|
||
</div>
|
||
<div class="clearer"></div>
|
||
</div>
|
||
<div class="related" role="navigation" aria-label="Related">
|
||
<h3>Navigation</h3>
|
||
<ul>
|
||
<li class="right" style="margin-right: 10px">
|
||
<a href="../genindex.html" title="General Index"
|
||
>index</a></li>
|
||
<li class="right" >
|
||
<a href="../py-modindex.html" title="Python Module Index"
|
||
>modules</a> |</li>
|
||
<li class="right" >
|
||
<a href="copyreg.html" title="copyreg — Register pickle support functions"
|
||
>next</a> |</li>
|
||
<li class="right" >
|
||
<a href="persistence.html" title="Data Persistence"
|
||
>previous</a> |</li>
|
||
|
||
<li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"/></li>
|
||
<li><a href="https://www.python.org/">Python</a> »</li>
|
||
<li class="switchers">
|
||
<div class="language_switcher_placeholder"></div>
|
||
<div class="version_switcher_placeholder"></div>
|
||
</li>
|
||
<li>
|
||
|
||
</li>
|
||
<li id="cpython-language-and-version">
|
||
<a href="../index.html">3.13.3 Documentation</a> »
|
||
</li>
|
||
|
||
<li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li>
|
||
<li class="nav-item nav-item-2"><a href="persistence.html" >Data Persistence</a> »</li>
|
||
<li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">pickle</span></code> — Python object serialization</a></li>
|
||
<li class="right">
|
||
|
||
|
||
<div class="inline-search" role="search">
|
||
<form class="inline-search" action="../search.html" method="get">
|
||
<input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box" />
|
||
<input type="submit" value="Go" />
|
||
</form>
|
||
</div>
|
||
|
|
||
</li>
|
||
<li class="right">
|
||
<label class="theme-selector-label">
|
||
Theme
|
||
<select class="theme-selector" oninput="activateTheme(this.value)">
|
||
<option value="auto" selected>Auto</option>
|
||
<option value="light">Light</option>
|
||
<option value="dark">Dark</option>
|
||
</select>
|
||
</label> |</li>
|
||
|
||
</ul>
|
||
</div>
|
||
<div class="footer">
|
||
©
|
||
<a href="../copyright.html">
|
||
|
||
Copyright
|
||
|
||
</a>
|
||
2001-2025, Python Software Foundation.
|
||
<br />
|
||
This page is licensed under the Python Software Foundation License Version 2.
|
||
<br />
|
||
Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License.
|
||
<br />
|
||
|
||
See <a href="/license.html">History and License</a> for more information.<br />
|
||
|
||
|
||
<br />
|
||
|
||
The Python Software Foundation is a non-profit corporation.
|
||
<a href="https://www.python.org/psf/donations/">Please donate.</a>
|
||
<br />
|
||
<br />
|
||
Last updated on Apr 08, 2025 (14:33 UTC).
|
||
|
||
<a href="/bugs.html">Found a bug</a>?
|
||
|
||
<br />
|
||
|
||
Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.
|
||
</div>
|
||
|
||
</body>
|
||
</html> |