PageSourceSearch

https://glaringlee.github.io/notes/broadcasting.html

html glaringlee.github.io collected 2026-10-03 09:14:45 UTC 41,844 bytes, 750 lines download raw bytes

1
2
3
4<!DOCTYPE html>
5<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
6<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
7<head>
8  <meta charset="utf-8">
9  
10  <meta name="viewport" content="width=device-width, initial-scale=1.0">
11  
12  <title>Broadcasting semantics &mdash; PyTorch master documentation</title>
13  
14
15  
16  
17  
18  
19    <link rel="canonical" href="https://pytorch.org/docs/stable/notes/broadcasting.html"/>
20  
21
22  
23
24  
25  
26    
27
28  
29
30  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
31  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
32  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" type="text/css" />
33  <link rel="stylesheet" href="../_static/css/jit.css" type="text/css" />
34  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" type="text/css" />
35  <link rel="stylesheet" href="../_static/katex-math.css" type="text/css" />
36    <link rel="index" title="Index" href="../genindex.html" />
37    <link rel="search" title="Search" href="../search.html" />
38    <link rel="next" title="CPU threading and TorchScript inference" href="cpu_threading_torchscript_inference.html" />
39    <link rel="prev" title="Autograd mechanics" href="autograd.html" /> 
40
41  
42  
42<script src="../_static/js/modernizr.min.js"></script>
42
43
44  <!-- Preload the theme fonts -->
45
46<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
47<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
48<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
49<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
50<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
51<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
52
53<!-- Preload the katex fonts -->
54
55<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
56<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
57<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
58<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
59<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
60<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
61<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
62<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
63</head>
64
65<div class="container-fluid header-holder tutorials-header" id="header-holder">
66  <div class="container">
67    <div class="header-container">
68      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
69
70      <div class="main-menu">
71        <ul>
72          <li>
73            <a href="https://pytorch.org/get-started">Get Started</a>
74          </li>
75
76          <li>
77            <div class="ecosystem-dropdown">
78              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
79                Ecosystem
80              </a>
81              <div class="ecosystem-dropdown-menu">
82                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
83                  <span class=dropdown-title>Models (Beta)</span>
84                  <p>Discover, publish, and reuse pre-trained models</p>
85                </a>
86                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
87                  <span class=dropdown-title>Tools & Libraries</span>
88                  <p>Explore the ecosystem of tools and libraries</p>
89                </a>
90              </div>
91            </div>
92          </li>
93
94          <li>
95            <a href="https://pytorch.org/mobile">Mobile</a>
96          </li>
97
98          <li>
99            <a href="https://pytorch.org/blog/">Blog</a>
100          </li>
101
102          <li>
103            <a href="https://pytorch.org/tutorials">Tutorials</a>
104          </li>
105
106          <li class="active">
107            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
108          </li>
109
110          <li>
111            <div class="resources-dropdown">
112              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
113                Resources
114              </a>
115              <div class="resources-dropdown-menu">
116                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
117                  <span class=dropdown-title>Developer Resources</span>
118                  <p>Find resources and get questions answered</p>
119                </a>
120                <a class="nav-dropdown-item" href="https://pytorch.org/features">
121                  <span class=dropdown-title>About</span>
122                  <p>Learn about PyTorch’s features and capabilities</p>
123                </a>
124              </div>
125            </div>
126          </li>
127
128          <li>
129            <a href="https://github.com/pytorch/pytorch">Github</a>
130          </li>
131        </ul>
132      </div>
133
134      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
135    </div>
136
137  </div>
138</div>
139
140
141<body class="pytorch-body">
142
143   
144
145    
146
147    <div class="table-of-contents-link-wrapper">
148      <span>Table of Contents</span>
149      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
150    </div>
151
152    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
153      <div class="pytorch-side-scroll">
154        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
155          <div class="pytorch-left-menu-search">
156            
157
158            
159              
160              
161                <div class="version">
162                  master (1.7.0a0+5ab5566 )
163                </div>
164              
165            
166
167            
168
169
170  
171
172
173<div role="search">
174  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
175    <input type="text" name="q" placeholder="Search Docs" />
176    <input type="hidden" name="check_keywords" value="yes" />
177    <input type="hidden" name="area" value="default" />
178  </form>
179</div>
180
181            
182          </div>
183
184          
185<div>
186  <a style="color:#F05732" href="https://pytorch.org/docs/stable/notes/broadcasting.html">
187    You are viewing unstable developer preview docs.
188    Click here to view docs for latest stable release.
189  </a>
190</div>
191
192            
193            
194              
195            
196            
197              <p class="caption"><span class="caption-text">Notes</span></p>
198<ul class="current">
199<li class="toctree-l1"><a class="reference internal" href="amp_examples.html">Automatic Mixed Precision examples</a></li>
200<li class="toctree-l1"><a class="reference internal" href="autograd.html">Autograd mechanics</a></li>
201<li class="toctree-l1 current"><a class="current reference internal" href="#">Broadcasting semantics</a></li>
202<li class="toctree-l1"><a class="reference internal" href="cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
203<li class="toctree-l1"><a class="reference internal" href="cuda.html">CUDA semantics</a></li>
204<li class="toctree-l1"><a class="reference internal" href="ddp.html">Distributed Data Parallel</a></li>
205<li class="toctree-l1"><a class="reference internal" href="extending.html">Extending PyTorch</a></li>
206<li class="toctree-l1"><a class="reference internal" href="faq.html">Frequently Asked Questions</a></li>
207<li class="toctree-l1"><a class="reference internal" href="large_scale_deployments.html">Features for large-scale deployments</a></li>
208<li class="toctree-l1"><a class="reference internal" href="multiprocessing.html">Multiprocessing best practices</a></li>
209<li class="toctree-l1"><a class="reference internal" href="randomness.html">Reproducibility</a></li>
210<li class="toctree-l1"><a class="reference internal" href="serialization.html">Serialization semantics</a></li>
211<li class="toctree-l1"><a class="reference internal" href="windows.html">Windows FAQ</a></li>
212</ul>
213<p class="caption"><span class="caption-text">Language Bindings</span></p>
214<ul>
215<li class="toctree-l1"><a class="reference internal" href="../cpp_index.html">C++</a></li>
216<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/javadoc/">Javadoc</a></li>
217</ul>
218<p class="caption"><span class="caption-text">Python API</span></p>
219<ul>
220<li class="toctree-l1"><a class="reference internal" href="../torch.html">torch</a></li>
221<li class="toctree-l1"><a class="reference internal" href="../nn.html">torch.nn</a></li>
222<li class="toctree-l1"><a class="reference internal" href="../nn.functional.html">torch.nn.functional</a></li>
223<li class="toctree-l1"><a class="reference internal" href="../tensors.html">torch.Tensor</a></li>
224<li class="toctree-l1"><a class="reference internal" href="../tensor_attributes.html">Tensor Attributes</a></li>
225<li class="toctree-l1"><a class="reference internal" href="../tensor_view.html">Tensor Views</a></li>
226<li class="toctree-l1"><a class="reference internal" href="../autograd.html">torch.autograd</a></li>
227<li class="toctree-l1"><a class="reference internal" href="../cuda.html">torch.cuda</a></li>
228<li class="toctree-l1"><a class="reference internal" href="../amp.html">torch.cuda.amp</a></li>
229<li class="toctree-l1"><a class="reference internal" href="../distributed.html">torch.distributed</a></li>
230<li class="toctree-l1"><a class="reference internal" href="../distributions.html">torch.distributions</a></li>
231<li class="toctree-l1"><a class="reference internal" href="../futures.html">torch.futures</a></li>
232<li class="toctree-l1"><a class="reference internal" href="../hub.html">torch.hub</a></li>
233<li class="toctree-l1"><a class="reference internal" href="../jit.html">torch.jit</a></li>
234<li class="toctree-l1"><a class="reference internal" href="../nn.init.html">torch.nn.init</a></li>
235<li class="toctree-l1"><a class="reference internal" href="../onnx.html">torch.onnx</a></li>
236<li class="toctree-l1"><a class="reference internal" href="../optim.html">torch.optim</a></li>
237<li class="toctree-l1"><a class="reference internal" href="../quantization.html">Quantization</a></li>
238<li class="toctree-l1"><a class="reference internal" href="../rpc.html">Distributed RPC Framework</a></li>
239<li class="toctree-l1"><a class="reference internal" href="../random.html">torch.random</a></li>
240<li class="toctree-l1"><a class="reference internal" href="../sparse.html">torch.sparse</a></li>
241<li class="toctree-l1"><a class="reference internal" href="../storage.html">torch.Storage</a></li>
242<li class="toctree-l1"><a class="reference internal" href="../bottleneck.html">torch.utils.bottleneck</a></li>
243<li class="toctree-l1"><a class="reference internal" href="../checkpoint.html">torch.utils.checkpoint</a></li>
244<li class="toctree-l1"><a class="reference internal" href="../cpp_extension.html">torch.utils.cpp_extension</a></li>
245<li class="toctree-l1"><a class="reference internal" href="../data.html">torch.utils.data</a></li>
246<li class="toctree-l1"><a class="reference internal" href="../dlpack.html">torch.utils.dlpack</a></li>
247<li class="toctree-l1"><a class="reference internal" href="../model_zoo.html">torch.utils.model_zoo</a></li>
248<li class="toctree-l1"><a class="reference internal" href="../tensorboard.html">torch.utils.tensorboard</a></li>
249<li class="toctree-l1"><a class="reference internal" href="../type_info.html">Type Info</a></li>
250<li class="toctree-l1"><a class="reference internal" href="../named_tensor.html">Named Tensors</a></li>
251<li class="toctree-l1"><a class="reference internal" href="../name_inference.html">Named Tensors operator coverage</a></li>
252<li class="toctree-l1"><a class="reference internal" href="../__config__.html">torch.__config__</a></li>
253</ul>
254<p class="caption"><span class="caption-text">Libraries</span></p>
255<ul>
256<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
257<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
258<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
259<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
260<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
261</ul>
262<p class="caption"><span class="caption-text">Community</span></p>
263<ul>
264<li class="toctree-l1"><a class="reference internal" href="../community/contribution_guide.html">
264PyTorch Contribution Guide</a></li>
265<li class="toctree-l1"><a class="reference internal" href="../community/governance.html">PyTorch Governance</a></li>
266<li class="toctree-l1"><a class="reference internal" href="../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
267</ul>
268
269            
270          
271
272        </div>
273      </div>
274    </nav>
275
276    <div class="pytorch-container">
277      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
278        <div class="pytorch-breadcrumbs-wrapper">
279          
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295<div role="navigation" aria-label="breadcrumbs navigation">
296
297  <ul class="pytorch-breadcrumbs">
298    
299      <li>
300        <a href="../index.html">
301          
302            Docs
303          
304        </a> &gt;
305      </li>
306
307        
308      <li>Broadcasting semantics</li>
309    
310    
311      <li class="pytorch-breadcrumbs-aside">
312        
313            
314            <a href="../_sources/notes/broadcasting.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
315          
316        
317      </li>
318    
319  </ul>
320
321  
322</div>
323        </div>
324
325        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
326          Shortcuts
327        </div>
328      </div>
329
330      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
331        <div class="pytorch-content-left">
332
333        
334          
335          <div class="rst-content">
336          
337            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
338             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
339              
340  <div class="section" id="broadcasting-semantics">
341<span id="id1"></span><h1>Broadcasting semantics<a class="headerlink" href="#broadcasting-semantics" title="Permalink to this headline">¶</a></h1>
342<p>Many PyTorch operations support <a class="reference external" href="https://numpy.org/doc/stable/user/basics.broadcasting.html#module-numpy.doc.broadcasting" title="(in NumPy v1.19)"><code class="xref any docutils literal notranslate"><span class="pre">NumPy</span> <span class="pre">Broadcasting</span> <span class="pre">Semantics</span></code></a>.</p>
343<p>In short, if a PyTorch operation supports broadcast, then its Tensor arguments can be
344automatically expanded to be of equal sizes (without making copies of the data).</p>
345<div class="section" id="general-semantics">
346<h2>General semantics<a class="headerlink" href="#general-semantics" title="Permalink to this headline">¶</a></h2>
347<p>Two tensors are “broadcastable” if the following rules hold:</p>
348<ul class="simple">
349<li><p>Each tensor has at least one dimension.</p></li>
350<li><p>When iterating over the dimension sizes, starting at the trailing dimension,
351the dimension sizes must either be equal, one of them is 1, or one of them
352does not exist.</p></li>
353</ul>
354<p>For Example:</p>
355<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">3</span><span class="p">)</span>
356<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">3</span><span class="p">)</span>
357<span class="go"># same shapes are always broadcastable (i.e. the above rules always hold)</span>
358
359<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">((</span><span class="mi">0</span><span class="p">,))</span>
360<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span>
361<span class="go"># x and y are not broadcastable, because x does not have at least 1 dimension</span>
362
363<span class="go"># can line up trailing dimensions</span>
364<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
365<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span>  <span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
366<span class="go"># x and y are broadcastable.</span>
367<span class="go"># 1st trailing dimension: both have size 1</span>
368<span class="go"># 2nd trailing dimension: y has size 1</span>
369<span class="go"># 3rd trailing dimension: x size == y size</span>
370<span class="go"># 4th trailing dimension: y dimension doesn&#39;t exist</span>
371
372<span class="go"># but:</span>
373<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
374<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span>  <span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
375<span class="go"># x and y are not broadcastable, because in the 3rd trailing dimension 2 != 3</span>
376</pre></div>
377</div>
378<p>If two tensors <code class="xref py py-attr docutils literal notranslate"><span class="pre">x</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">y</span></code> are “broadcastable”, the resulting tensor size
379is calculated as follows:</p>
380<ul class="simple">
381<li><p>If the number of dimensions of <code class="xref py py-attr docutils literal notranslate"><span class="pre">x</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">y</span></code> are not equal, prepend 1
382to the dimensions of the tensor with fewer dimensions to make them equal length.</p></li>
383<li><p>Then, for each dimension size, the resulting dimension size is the max of the sizes of
384<code class="xref py py-attr docutils literal notranslate"><span class="pre">x</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">y</span></code> along that dimension.</p></li>
385</ul>
386<p>For Example:</p>
387<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># can line up trailing dimensions to make reading easier</span>
388<span class="o">&gt;&gt;&gt;</span> <span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
389<span class="o">&gt;&gt;&gt;</span> <span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span>  <span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
390<span class="o">&gt;&gt;&gt;</span> <span class="p">(</span><span class="n">x</span><span class="o">+</span><span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
391<span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
392
393<span class="c1"># but not necessary:</span>
394<span class="o">&gt;&gt;&gt;</span> <span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
395<span class="o">&gt;&gt;&gt;</span> <span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">7</span><span class="p">)</span>
396<span class="o">&gt;&gt;&gt;</span> <span class="p">(</span><span class="n">x</span><span class="o">+</span><span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
397<span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">([</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">])</span>
398
399<span class="o">&gt;&gt;&gt;</span> <span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
400<span class="o">&gt;&gt;&gt;</span> <span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
401<span class="o">&gt;&gt;&gt;</span> <span class="p">(</span><span class="n">x</span><span class="o">+</span><span class="n">y</span><span class="p">)</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
402<span class="ne">RuntimeError</span><span class="p">:</span> <span class="n">The</span> <span class="n">size</span> <span class="n">of</span> <span class="n">tensor</span> <span class="n">a</span> <span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="n">must</span> <span class="n">match</span> <span class="n">the</span> <span class="n">size</span> <span class="n">of</span> <span class="n">tensor</span> <span class="n">b</span> <span class="p">(</span><span class="mi">3</span><span class="p">)</span> <span class="n">at</span> <span class="n">non</span><span class="o">-</span><span class="n">singleton</span> <span class="n">dimension</span> <span class="mi">1</span>
403</pre></div>
404</div>
405</div>
406<div class="section" id="in-place-semantics">
407<h2>In-place semantics<a class="headerlink" href="#in-place-semantics" title="Permalink to this headline">¶</a></h2>
408<p>One complication is that in-place operations do not allow the in-place tensor to change shape
409as a result of the broadcast.</p>
410<p>For Example:</p>
411<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
412<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
413<span class="gp">&gt;&gt;&gt; </span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">
413add_</span><span class="p">(</span><span class="n">y</span><span class="p">))</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
414<span class="go">torch.Size([5, 3, 4, 1])</span>
415
416<span class="go"># but:</span>
417<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
418<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">7</span><span class="p">)</span>
419<span class="gp">&gt;&gt;&gt; </span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">add_</span><span class="p">(</span><span class="n">y</span><span class="p">))</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
420<span class="go">RuntimeError: The expanded size of the tensor (1) must match the existing size (7) at non-singleton dimension 2.</span>
421</pre></div>
422</div>
423</div>
424<div class="section" id="backwards-compatibility">
425<h2>Backwards compatibility<a class="headerlink" href="#backwards-compatibility" title="Permalink to this headline">¶</a></h2>
426<p>Prior versions of PyTorch allowed certain pointwise functions to execute on tensors with different shapes,
427as long as the number of elements in each tensor was equal.  The pointwise operation would then be carried
428out by viewing each tensor as 1-dimensional.  PyTorch now supports broadcasting and the “1-dimensional”
429pointwise behavior is considered deprecated and will generate a Python warning in cases where tensors are
430not broadcastable, but have the same number of elements.</p>
431<p>Note that the introduction of broadcasting can cause backwards incompatible changes in the case where
432two tensors do not have the same shape, but are broadcastable and have the same number of elements.
433For Example:</p>
434<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">4</span><span class="p">))</span>
435</pre></div>
436</div>
437<p>would previously produce a Tensor with size: torch.Size([4,1]), but now produces a Tensor with size: torch.Size([4,4]).
438In order to help identify cases in your code where backwards incompatibilities introduced by broadcasting may exist,
439you may set <cite>torch.utils.backcompat.broadcast_warning.enabled</cite> to <cite>True</cite>, which will generate a python warning
440in such cases.</p>
441<p>For Example:</p>
442<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">backcompat</span><span class="o">.</span><span class="n">broadcast_warning</span><span class="o">.</span><span class="n">enabled</span><span class="o">=</span><span class="kc">True</span>
443<span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span><span class="mi">1</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">4</span><span class="p">))</span>
444<span class="go">__main__:1: UserWarning: self and other do not have the same shape, but are broadcastable, and have the same number of elements.</span>
445<span class="go">Changing behavior in a backwards incompatible manner to broadcasting rather than viewing as 1-dimensional.</span>
446</pre></div>
447</div>
448</div>
449</div>
450
451
452             </article>
453             
454            </div>
455            <footer>
456  
457    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
458      
459        <a href="cpu_threading_torchscript_inference.html" class="btn btn-neutral float-right" title="CPU threading and TorchScript inference" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
460      
461      
462        <a href="autograd.html" class="btn btn-neutral" title="Autograd mechanics" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
463      
464    </div>
465  
466
467  
468
469    <hr>
470
471  
472
473  <div role="contentinfo">
474    <p>
475        &copy;
475 Copyright 2019, Torch Contributors.
476
477    </p>
478  </div>
479    
480      <div>
481        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
482      </div>
483     
484
485</footer>
486
487          </div>
488        </div>
489
490        <div class="pytorch-content-right" id="pytorch-content-right">
491          <div class="pytorch-right-menu" id="pytorch-right-menu">
492            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
493              <ul>
494<li><a class="reference internal" href="#">Broadcasting semantics</a><ul>
495<li><a class="reference internal" href="#general-semantics">General semantics</a></li>
496<li><a class="reference internal" href="#in-place-semantics">In-place semantics</a></li>
497<li><a class="reference internal" href="#backwards-compatibility">Backwards compatibility</a></li>
498</ul>
499</li>
500</ul>
501
502            </div>
503          </div>
504        </div>
505      </section>
506    </div>
507
508  
509
510
511  
512
513     
514       
514<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
514
515         
515<script src="../_static/jquery.js"></script>
515
516         
516<script src="../_static/underscore.js"></script>
516
517         
517<script src="../_static/doctools.js"></script>
517
518         
518<script src="../_static/language_data.js"></script>
518
519     
520
521  
522
523  
523<script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
523
524  
524<script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
524
525  
525<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
525
526  
526<script type="text/javascript" src="../_static/js/theme.js"></script>
526
527
528  
528<script type="text/javascript">
529      jQuery(function () {
530          SphinxRtdTheme.Navigation.enable(true);
531      });
532  </script>
532
533 
534<script>
vendor: 345 bytes, lines 534-540
534
535  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
536  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
537  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
538  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
539
540  ga('create', '
540UA-90545585-1
vendor: 39 bytes, lines 540-542
540', 'auto');
541  ga('send', 'pageview');
542
543</script>
vendor: 65 bytes, lines 543-545
543
544
545<script async src="https://www.googletagmanager.com/gtag/js?id=
545UA-117752657-2
vendor: 13 bytes, lines 545-546
545"></script>
546
547<script>
548  
vendor: 135 bytes, lines 548-553
548window.dataLayer = window.dataLayer || [];
549
550  function gtag(){dataLayer.push(arguments);}
551
552  gtag('js', new Date());
553  gtag('config', '
553UA-117752657-2
vendor: 4 bytes, line 553
553');
554</script>
554
555
556<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>
557
558
559  <!-- Begin Footer -->
560
561  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
562    <div class="container">
563      <div class="row">
564        <div class="col-md-4 text-center">
565          <h2>Docs</h2>
566          <p>Access comprehensive developer documentation for PyTorch</p>
567          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
568        </div>
569
570        <div class="col-md-4 text-center">
571          <h2>Tutorials</h2>
572          <p>Get in-depth tutorials for beginners and advanced developers</p>
573          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
574        </div>
575
576        <div class="col-md-4 text-center">
577          <h2>Resources</h2>
578          <p>Find development resources and get your questions answered</p>
579          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
580        </div>
581      </div>
582    </div>
583  </div>
584
585  <footer class="site-footer">
586    <div class="container footer-container">
587      <div class="footer-logo-wrapper">
588        <a href="https://pytorch.org/" class="footer-logo"></a>
589      </div>
590
591      <div class="footer-links-wrapper">
592        <div class="footer-links-col">
593          <ul>
594            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
595            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
596            <li><a href="https://pytorch.org/features">Features</a></li>
597            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
598            <li><a href="https://pytorch.org/blog/">Blog</a></li>
599            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
600          </ul>
601        </div>
602
603        <div class="footer-links-col">
604          <ul>
605            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
606            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
607            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
608            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
609            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
610            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
611          </ul>
612        </div>
613
614        <div class="footer-links-col follow-us-col">
615          <ul>
616            <li class="list-title">Stay Connected</li>
617            <li>
618              <div id="mc_embed_signup">
619                <form
620                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
621                  method="post"
622                  id="mc-embedded-subscribe-form"
623                  name="mc-embedded-subscribe-form"
624                  class="email-subscribe-form validate"
625                  target="_blank"
626                  novalidate>
627                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
628                    <div class="mc-field-group">
629                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
630                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
631                    </div>
632
633                    <div id="mce-responses" class="clear">
634                      <div class="response" id="mce-error-response" style="display:none"></div>
635                      <div class="response" id="mce-success-response" style="display:none"></div>
636                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->
637
638                    <div style="position: absolute; left: -5000px;" aria-hidden="true">
638<input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>
639
640                    <div class="clear">
641                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
642                    </div>
643                  </div>
644                </form>
645              </div>
646
647            </li>
648          </ul>
649
650          <div class="footer-social-icons">
651            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
652            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
653            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
654          </div>
655        </div>
656      </div>
657    </div>
658  </footer>
659
660  <div class="cookie-banner-wrapper">
661  <div class="container">
662    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
663    <img class="close-button" src="../_static/images/pytorch-x.svg">
664  </div>
665</div>
666
667  <!-- End Footer -->
668
669  <!-- Begin Mobile Menu -->
670
671  <div class="mobile-main-menu">
672    <div class="container-fluid">
673      <div class="container">
674        <div class="mobile-main-menu-header-container">
675          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
676          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
677        </div>
678      </div>
679    </div>
680
681    <div class="mobile-main-menu-links-container">
682      <div class="main-menu">
683        <ul>
684          <li>
685            <a href="https://pytorch.org/get-started">Get Started</a>
686          </li>
687
688          <li>
689            <a href="https://pytorch.org/features">Features</a>
690          </li>
691
692          <li>
693            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
694          </li>
695
696          <li>
697            <a href="https://pytorch.org/mobile">Mobile</a>
698          </li>
699
700          <li>
701            <a href="https://pytorch.org/hub">PyTorch Hub</a>
702          </li>
703
704          <li>
705            <a href="https://pytorch.org/blog/">Blog</a>
706          </li>
707
708          <li>
709            <a href="https://pytorch.org/tutorials">Tutorials</a>
710          </li>
711
712          <li class="active">
713            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
714          </li>
715
716          <li>
717            <a href="https://pytorch.org/resources">Resources</a>
718          </li>
719
720          <li>
721            <a href="https://github.com/pytorch/pytorch">Github</a>
722          </li>
723        </ul>
724      </div>
725    </div>
726  </div>
727
728  <!-- End Mobile Menu -->
729
730  
730<script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>
730
731
732  
732<script type="text/javascript">
733    $(document).ready(function() {
734      mobileMenu.bind();
735      mobileTOC.bind();
736      pytorchAnchors.bind();
737      sideMenus.bind();
738      scrollToAnchor.bind();
739      highlightNavigation.bind();
740      mainMenuDropdown.bind();
741      filterTags.bind();
742
743      // Add class to links that have code blocks, since we cannot create links in code blocks
744      $("article.pytorch-article a span.pre").each(function(e) {
745        $(this).closest("a").addClass("has-code");
746      });
747    })
748  </script>
748
749</body>
750</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.