upload

app.py

@@ -1,4 +1,4 @@
-import imp
+# import imp
 import streamlit as st
 import pandas as pd
 import numpy as np
@@ -41,51 +41,51 @@ if crawling_or_not:
     # cleaning the pdf lists
     pdf_lists = [i.strip() for i in pdf_lists if len(i) > 0]
     # TODO: limit the number of paper up to 10 since I am not sure that whether base64 support large file download
-    try: 
-        if len(pdf_lists) > predefined_limits:
-            st.warning(f"Currently only support up to {predefined_limits} papers. Please input less than {predefined_limits} papers.")
-        else:
-            # parsing
-            base='./download/'
-            project_name = get_timestamp().replace(" ","-")
-            base = os.path.join(base,project_name)
-            make_dir_if_not_exist(base)
-            
-            # st.write(download_status)
-            with st.spinner("Downloading papers..."):
-                # progress bar
-                bar = st.progress(0)
-                download_status = st.empty()
-                N = len(pdf_lists)
-                for i, pdf_link in tqdm(enumerate(pdf_lists)):
-                    title = get_name_from_arvix(pdf_link)
-                    file_stamp = pdf_link.split("/")[-1]
-                    source_link = "https://arxiv.org/e-print/"+file_stamp
-                    inp = os.path.join(base,'input')
-                    make_dir_if_not_exist(inp)
-                    out = os.path.join(base,'output')
-                    make_dir_if_not_exist(out)
-                    response = requests.get(source_link)
-                    filename = file_stamp+".tar.gz"
-                    filepath = os.path.join(inp,filename)
-                    open(filepath, "wb").write(response.content)
-                    outpath = os.path.join(out,title)
-                    untar(filepath,outpath)
-                    
-                    # finish one paper
-                    bar.progress((i+1)/N)
-                    download_status.text(f"Iteration [{i+1}/{N}]: Finish Downloading of "+title)
-            
-            with st.spinner("Archiving as Zip Files..."):
-                # save it as zip file
-                filepath = archive_dir(out,os.path.join(base,project_name))
-
-                # download
-                b64 = ToBase64(filepath).decode()
-            href = f"<a href='data:file/csv;base64,{b64}' download='arxiv2latex-output-{datetime.datetime.now()}.zip' color='red'>Click here to Download the Output Latex Zip Files</a>"
-            st.markdown(href, unsafe_allow_html=True)
+    # try: 
+    if len(pdf_lists) > predefined_limits:
+        st.warning(f"Currently only support up to {predefined_limits} papers. Please input less than {predefined_limits} papers.")
+    else:
+        # parsing
+        base='./download/'
+        project_name = get_timestamp().replace(" ","-")
+        base = os.path.join(base,project_name)
+        make_dir_if_not_exist(base)
+        
+        # st.write(download_status)
+        with st.spinner("Downloading papers..."):
+            # progress bar
+            bar = st.progress(0)
+            download_status = st.empty()
+            N = len(pdf_lists)
+            for i, pdf_link in tqdm(enumerate(pdf_lists)):
+                title = get_name_from_arvix(pdf_link)
+                file_stamp = pdf_link.split("/")[-1]
+                source_link = "https://arxiv.org/e-print/"+file_stamp
+                inp = os.path.join(base,'input')
+                make_dir_if_not_exist(inp)
+                out = os.path.join(base,'output')
+                make_dir_if_not_exist(out)
+                response = requests.get(source_link)
+                filename = file_stamp+".tar.gz"
+                filepath = os.path.join(inp,filename)
+                open(filepath, "wb").write(response.content)
+                outpath = os.path.join(out,title)
+                untar(filepath,outpath)
                 
-            # 状态
-            st.success("Finished")
-    except Exception as e:
-        st.error("Something goes wrong. Please check the input or concat me to fix this bug. Error message: \n"+str(e))
+                # finish one paper
+                bar.progress((i+1)/N)
+                download_status.text(f"Iteration [{i+1}/{N}]: Finish Downloading of "+title)
+        
+        with st.spinner("Archiving as Zip Files..."):
+            # save it as zip file
+            filepath = archive_dir(out,os.path.join(base,project_name))
+
+            # download
+            b64 = ToBase64(filepath).decode()
+        href = f"<a href='data:file/csv;base64,{b64}' download='arxiv2latex-output-{datetime.datetime.now()}.zip' color='red'>Click here to Download the Output Latex Zip Files</a>"
+        st.markdown(href, unsafe_allow_html=True)
+            
+        # 状态
+        st.success("Finished")
+    # except Exception as e:
+    #     st.error("Something goes wrong. Please check the input or concat me to fix this bug. Error message: \n"+str(e))

download/20230719163556/20230719163556.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c08fed6381f25599036729607317c42b5c1a23ffc648cd0d43e3087819c4ef
+size 1719685

download/20230719163556/input/1512.03385.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a975e8b798fb4c447551bffd4e74a3abf169b151e4435390ab6d16bf88cef880
+size 506177

download/20230719163556/input/1706.03762.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2500348e730741b9a8eafcac0301f7e3f0b1c670f9e08ce62e6c45a36a64cf0a
+size 1150961

download/20230719163556/output/Attention-Is-All-You-Need/background.tex

@@ -0,0 +1,58 @@
+The goal of reducing sequential computation also forms the foundation of the Extended Neural GPU \citep{extendedngpu}, ByteNet \citep{NalBytenet2017} and ConvS2S \citep{JonasFaceNet2017}, all of which use convolutional neural networks as basic building block, computing hidden representations in parallel for all input and output positions. In these models, the number of operations required to relate signals from two arbitrary input or output positions grows in the distance between positions, linearly for ConvS2S and logarithmically for ByteNet. This makes it more difficult to learn dependencies between distant positions \citep{hochreiter2001gradient}. In the Transformer this is reduced to a constant number of operations, albeit at the cost of reduced effective resolution due to averaging attention-weighted positions, an effect we counteract with Multi-Head Attention as described in section~\ref{sec:attention}. 
+
+Self-attention, sometimes called intra-attention is an attention mechanism relating different positions of a single sequence in order to compute a representation of the sequence. Self-attention has been used successfully in a variety of tasks including reading comprehension, abstractive summarization, textual entailment and learning task-independent sentence representations \citep{cheng2016long, decomposableAttnModel, paulus2017deep, lin2017structured}.
+
+End-to-end memory networks are based on a recurrent attention mechanism instead of sequence-aligned recurrence and have been shown to perform well on simple-language question answering and language modeling tasks \citep{sukhbaatar2015}.
+
+To the best of our knowledge, however, the Transformer is the first transduction model relying entirely on self-attention to compute representations of its input and output without using sequence-aligned RNNs or convolution.
+In the following sections, we will describe the Transformer, motivate self-attention and discuss its advantages over models such as \citep{neural_gpu, NalBytenet2017} and \citep{JonasFaceNet2017}.
+
+
+%\citep{JonasFaceNet2017} report new SOTA on machine translation for English-to-German (EnDe), Enlish-to-French (EnFr) and English-to-Romanian language pairs. 
+
+%For example,! in MT, we must draw information from both input and previous output words to translate an output word accurately. An attention layer \citep{bahdanau2014neural} can connect a very large number of positions at low computation cost, making it an essential ingredient in competitive recurrent models for machine translation.
+
+%A natural question to ask then is, "Could we replace recurrence with attention?". \marginpar{Don't know if it's the most natural question to ask given the previous statements. Also, need to say that the complexity table summarizes these statements} Such a model would be blessed with the computational efficiency of attention and the power of cross-positional communication. In this work, show that pure attention models work remarkably well for MT, achieving new SOTA results on EnDe and EnFr, and can be trained in under $2$ days on xyz architecture. 
+
+%After the seminal models introduced in \citep{sutskever14, bahdanau2014neural, cho2014learning}, recurrent models have become the dominant solution for both sequence modeling and sequence-to-sequence transduction. Many efforts such as \citep{wu2016google,luong2015effective,jozefowicz2016exploring} have pushed the boundaries of machine translation (MT) and language modeling with recurrent endoder-decoder and recurrent language models. Recent effort \citep{shazeer2017outrageously} has successfully combined the power of conditional computation with sequence models to train very large models for MT, pushing SOTA at lower computational cost.
+
+%Recurrent models compute a vector of hidden states $h_t$, for each time step $t$ of computation. $h_t$ is a function of both the input at time $t$ and the previous hidden state $h_t$. This dependence on the previous hidden state precludes processing all timesteps at once, instead requiring long sequences of sequential operations.  In practice, this results in greatly reduced computational efficiency, as on modern computing hardware, a single operation on a large batch is much faster than a large number of operations on small batches.  The problem gets worse at longer sequence lengths. Although sequential computation is not a severe bottleneck at inference time, as autoregressively generating each output requires all previous outputs, the inability to compute scores at all output positions at once hinders us from rapidly training our models over large datasets. Although impressive work such as \citep{Kuchaiev2017Factorization} is able to significantly accelerate the training of LSTMs with factorization tricks, we are still bound by the linear dependence on sequence length.
+
+%If the model could compute hidden states at each time step using only the inputs and outputs,  it would be liberated from the dependence on results from previous time steps during training. This line of thought is the foundation of recent efforts such as the Markovian neural GPU \citep{neural_gpu}, ByteNet \citep{NalBytenet2017} and ConvS2S \citep{JonasFaceNet2017}, all of which use convolutional neural networks as a building block to compute hidden representations simultaneously for all timesteps, resulting in $O(1)$ sequential time complexity. \citep{JonasFaceNet2017} report new SOTA on machine translation for English-to-German (EnDe), Enlish-to-French (EnFr) and English-to-Romanian language pairs. 
+
+%A crucial component for accurate sequence prediction is modeling cross-positional communication. For example, in MT, we must draw information from both input and previous output words to translate an output word accurately. An attention layer \citep{bahdanau2014neural} can connect a very large number of positions at a low computation cost, also $O(1)$ sequential time complexity, making it an essential ingredient in recurrent encoder-decoder architectures for MT. A natural question to ask then is, "Could we replace recurrence with attention?". \marginpar{Don't know if it's the most natural question to ask given the previous statements. Also, need to say that the complexity table summarizes these statements} Such a model would be blessed with the computational efficiency of attention and the power of cross-positional communication. In this work, show that pure attention models work remarkably well for MT, achieving new SOTA results on EnDe and EnFr, and can be trained in under $2$ days on xyz architecture. 
+
+
+
+%Note: Facebook model is no better than RNNs in this regard, since it requires a number of layers proportional to the distance you want to communicate.  Bytenet is more promising, since it requires a logarithmnic number of layers (does bytenet have SOTA results)?   
+
+%Note: An attention  layer can connect a very large number of positions at a low computation cost in O(1) sequential operations.  This is why encoder-decoder attention has been so successful in seq-to-seq models so far.  It is only natural, then, to also use attention to connect the timesteps of the same sequence.
+
+%Note: I wouldn't say that long sequences are not a problem during inference.  It would be great if we could infer with no long sequences.  We could just say later on that, while our training graph is constant-depth, our model still requires sequential operations in the decoder part during inference due to the autoregressive nature of the model.   
+
+%\begin{table}[h!]
+%\caption{Attention models are quite efficient for cross-positional communications when sequence length is smaller than channel depth. $n$ represents the sequence length and $d$ represents the channel depth.}
+%\label{tab:op_complexities}
+%\begin{center}
+%\vspace{-5pt}
+%\scalebox{0.75}{
+
+%\begin{tabular}{l|c|c|c}
+%\hline \hline
+%Layer Type & Receptive & Complexity & Sequential  \\
+%           & Field     &            & Operations  \\
+%\hline
+%Pointwise Feed-Forward & $1$ & $O(n \cdot d^2)$ & $O(1)$ \\
+%\hline
+%Recurrent & $n$ & $O(n \cdot d^2)$ & $O(n)$ \\
+%\hline
+%Convolutional & $r$ & $O(r \cdot n \cdot d^2)$ & $O(1)$ \\
+%\hline
+%Convolutional (separable) & $r$ & $O(r \cdot n \cdot d + n %\cdot d^2)$ & $O(1)$ \\
+%\hline
+%Attention & $r$ & $O(r \cdot n \cdot d)$ & $O(1)$ \\
+%\hline \hline
+%\end{tabular}
+%}
+%\end{center}
+%\end{table}

download/20230719163556/output/Attention-Is-All-You-Need/introduction.tex

@@ -0,0 +1,18 @@
+Recurrent neural networks, long short-term memory \citep{hochreiter1997} and gated recurrent \citep{gruEval14} neural networks in particular, have been firmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation \citep{sutskever14, bahdanau2014neural, cho2014learning}. Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures \citep{wu2016google,luong2015effective,jozefowicz2016exploring}.
+
+Recurrent models typically factor computation along the symbol positions of the input and output sequences. Aligning the positions to steps in computation time, they generate a sequence of hidden states $h_t$, as a function of the previous hidden state $h_{t-1}$ and the input for position $t$. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples.
+%\marginpar{not sure if the memory constraints are understandable here}
+Recent work has achieved significant improvements in computational efficiency through factorization tricks \citep{Kuchaiev2017Factorization} and conditional computation \citep{shazeer2017outrageously}, while also improving model performance in case of the latter. The fundamental constraint of sequential computation, however, remains.
+
+%\marginpar{@all: there is work on analyzing what attention really does in seq2seq models, couldn't find it right away} 
+
+Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences \citep{bahdanau2014neural, structuredAttentionNetworks}. In all but a few cases \citep{decomposableAttnModel}, however, such attention mechanisms are used in conjunction with a recurrent network.
+
+%\marginpar{not sure if "cross-positional communication" is understandable without explanation}
+%\marginpar{insert exact training times and stats for the model that reaches sota earliest, maybe even a single GPU model?}
+
+In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs. 
+%\marginpar{you removed the constant number of repetitions part. I wrote it because I wanted to make it clear that the model does not only perform attention once, while it's also not recurrent. I thought that might be important to get across early.}
+
+% Just a standard paragraph with citations, rewrite.
+%After the seminal papers of \citep{sutskever14}, \citep{bahdanau2014neural}, and \citep{cho2014learning}, recurrent models have become the dominant solution for both sequence modeling and sequence-to-sequence transduction. Many efforts such as \citep{wu2016google,luong2015effective,jozefowicz2016exploring} have pushed the boundaries of machine translation and language modeling with recurrent sequence models. Recent effort \citep{shazeer2017outrageously} has combined the power of conditional computation with sequence models to train very large models for machine translation, pushing SOTA at lower computational cost. Recurrent models compute a vector of hidden states $h_t$, for each time step $t$ of computation. $h_t$ is a function of both the input at time $t$ and the previous hidden state $h_t$. This dependence on the previous hidden state encumbers recurrnet models to process multiple inputs at once, and their time complexity is a linear function of the length of the input and output, both during training and inference. [What I want to say here is that although this is fine during decoding, at training time, we are given both input and output and this linear nature does not allow the RNN to process all inputs and outputs simultaneously and haven't been used on datasets that are the of the scale of the web. What's the largest dataset we have ? . Talk about Nividia and possibly other's effors to speed up things, and possibly other efforts that alleviate this, but are still limited by it's comptuational nature]. Rest of the intro: What if you could construct the state based on the actual inputs and outputs, then you could construct them all at once. This has been the foundation of many promising recent efforts, bytenet,facenet (Also talk about quasi rnn here). Now we talk about attention!! Along with cell architectures such as long short-term meory (LSTM) \citep{hochreiter1997}, and gated recurrent units (GRUs) \citep{cho2014learning}, attention has emerged as an essential ingredient in successful sequence models, in particular for machine translation. In recent years, many, if not all, state-of-the-art (SOTA) results in machine translation have been achieved with attention-based sequence models \citep{wu2016google,luong2015effective,jozefowicz2016exploring}. Talk about the neon work on how it played with attention to do self attention! Then talk about what we do.

download/20230719163556/output/Attention-Is-All-You-Need/model_architecture.tex

@@ -0,0 +1,155 @@
+
+\begin{figure}
+  \centering
+  \includegraphics[scale=0.6]{Figures/ModalNet-21}
+  \caption{The Transformer - model architecture.}
+  \label{fig:model-arch}
+\end{figure}
+
+% Although the primary workhorse of our model is attention, 
+%Our model maintains the encoder-decoder structure that is common to many so-called sequence-to-sequence models \citep{bahdanau2014neural,sutskever14}.  As in all such architectures, the encoder computes a representation of the input sequence, and the decoder consumes these representations along with the output tokens to autoregressively produce the output sequence.  Where, traditionally, the encoder and decoder contain stacks of recurrent or convolutional layers, our encoder and decoder stacks are composed of attention layers and position-wise feed-forward layers (Figure~\ref{fig:model-arch}).  The following sections describe the gross architecture and these particular components in detail.
+
+Most competitive neural sequence transduction models have an encoder-decoder structure \citep{cho2014learning,bahdanau2014neural,sutskever14}. Here, the encoder maps an input sequence of symbol representations $(x_1, ..., x_n)$ to a sequence of continuous representations $\mathbf{z} = (z_1, ..., z_n)$. Given $\mathbf{z}$, the decoder then generates an output sequence $(y_1,...,y_m)$ of symbols one element at a time. At each step the model is auto-regressive \citep{graves2013generating}, consuming the previously generated symbols as additional input when generating the next.
+
+The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers for both the encoder and decoder, shown in the left and right halves of Figure~\ref{fig:model-arch}, respectively.
+
+\subsection{Encoder and Decoder Stacks}
+
+\paragraph{Encoder:}The encoder is composed of a stack of $N=6$ identical layers. Each layer has two sub-layers. The first is a multi-head self-attention mechanism, and the second is a simple, position-wise fully connected feed-forward network.   We employ a residual connection \citep{he2016deep} around each of the two sub-layers, followed by layer normalization \cite{layernorm2016}.  That is, the output of each sub-layer is $\mathrm{LayerNorm}(x + \mathrm{Sublayer}(x))$, where $\mathrm{Sublayer}(x)$ is the function implemented by the sub-layer itself.  To facilitate these residual connections, all sub-layers in the model, as well as the embedding layers, produce outputs of dimension $\dmodel=512$.
+
+\paragraph{Decoder:}The decoder is also composed of a stack of $N=6$ identical layers.  In addition to the two sub-layers in each encoder layer, the decoder inserts a third sub-layer, which performs multi-head attention over the output of the encoder stack.  Similar to the encoder, we employ residual connections around each of the sub-layers, followed by layer normalization.  We also modify the self-attention sub-layer in the decoder stack to prevent positions from attending to subsequent positions.  This masking, combined with fact that the output embeddings are offset by one position, ensures that the predictions for position $i$ can depend only on the known outputs at positions less than $i$.
+
+% In our model (Figure~\ref{fig:model-arch}), the encoder and decoder are composed of stacks of alternating self-attention layers (for cross-positional communication) and position-wise feed-forward layers (for in-place computation).  In addition, the decoder stack contains encoder-decoder attention layers.  Since attention is agnostic to the distances between words, our model requires a "positional encoding" to be added to the encoder and decoder input. The following sections describe all of these components in detail.
+
+\subsection{Attention} \label{sec:attention}
+An attention function can be described as mapping a query and a set of key-value pairs to an output, where the query, keys, values, and output are all vectors.  The output is computed as a weighted sum of the values, where the weight assigned to each value is computed by a compatibility function of the query with the corresponding key.
+
+\subsubsection{Scaled Dot-Product Attention} \label{sec:scaled-dot-prod}
+
+% \begin{figure}
+%   \centering
+%   \includegraphics[scale=0.6]{Figures/ModalNet-19}
+%   \caption{Scaled Dot-Product Attention.}
+%   \label{fig:multi-head-att}
+% \end{figure}
+
+We call our particular attention "Scaled Dot-Product Attention" (Figure~\ref{fig:multi-head-att}).   The input consists of queries and keys of dimension $d_k$, and values of dimension $d_v$.  We compute the dot products of the query with all keys, divide each by $\sqrt{d_k}$, and apply a softmax function to obtain the weights on the values.
+
+In practice, we compute the attention function on a set of queries simultaneously, packed together into a matrix $Q$.   The keys and values are also packed together into matrices $K$ and $V$.  We compute the matrix of outputs as:
+
+\begin{equation}
+   \mathrm{Attention}(Q, K, V) = \mathrm{softmax}(\frac{QK^T}{\sqrt{d_k}})V
+\end{equation}
+
+The two most commonly used attention functions are additive attention \citep{bahdanau2014neural}, and dot-product (multiplicative) attention.  Dot-product attention is identical to our algorithm, except for the scaling factor of $\frac{1}{\sqrt{d_k}}$. Additive attention computes the compatibility function using a feed-forward network with a single hidden layer.  While the two are similar in theoretical complexity, dot-product attention is much faster and more space-efficient in practice, since it can be implemented using highly optimized matrix multiplication code. 
+
+%We scale the dot products by $1/\sqrt{d_k}$ to limit the magnitude of the dot products, which works well in practice. Otherwise, we found applying the softmax to often result in weights very close to 0 or 1, and hence minuscule gradients.
+
+% Already described in the subsequent section
+%When used as part of decoder self-attention, an optional mask function is applied just before the softmax to prevent positions from attending to subsequent positions.   This mask simply sets the logits corresponding to all illegal connections (those outside of the lower triangle) to $-\infty$.
+
+%\paragraph{Comparison to Additive Attention: } We choose dot product attention over additive attention \citep{bahdanau2014neural} since it can be computed using highly optimized matrix multiplication code.  This optimization is particularly important to us, as we employ many attention layers in our model.
+
+While for small values of $d_k$ the two mechanisms perform similarly, additive attention outperforms dot product attention without scaling for larger values of $d_k$ \citep{DBLP:journals/corr/BritzGLL17}. We suspect that for large values of $d_k$, the dot products grow large in magnitude, pushing the softmax function into regions where it has extremely small gradients  \footnote{To illustrate why the dot products get large, assume that the components of $q$ and $k$ are independent random variables with mean $0$ and variance $1$.  Then their dot product, $q \cdot k = \sum_{i=1}^{d_k} q_ik_i$, has mean $0$ and variance $d_k$.}. To counteract this effect, we scale the dot products by $\frac{1}{\sqrt{d_k}}$.
+
+
+%We suspect this to be caused by the dot products growing too large in magnitude to result in useful gradients after applying the softmax function.  To counteract this, we scale the dot product by $1/\sqrt{d_k}$.
+
+
+\subsubsection{Multi-Head Attention} \label{sec:multihead}
+
+\begin{figure}
+\begin{minipage}[t]{0.5\textwidth}
+  \centering
+  Scaled Dot-Product Attention \\
+  \vspace{0.5cm}
+  \includegraphics[scale=0.6]{Figures/ModalNet-19}
+\end{minipage}
+\begin{minipage}[t]{0.5\textwidth}
+  \centering 
+  Multi-Head Attention \\
+  \vspace{0.1cm}
+  \includegraphics[scale=0.6]{Figures/ModalNet-20}  
+\end{minipage}
+
+
+  % \centering
+
+  \caption{(left) Scaled Dot-Product Attention. (right) Multi-Head Attention consists of several attention layers running in parallel.}
+  \label{fig:multi-head-att}
+\end{figure}
+
+Instead of performing a single attention function with $\dmodel$-dimensional keys, values and queries, we found it beneficial to linearly project the queries, keys and values $h$ times with different, learned linear projections to $d_k$, $d_k$ and $d_v$ dimensions, respectively.
+On each of these projected versions of queries, keys and values we then perform the attention function in parallel, yielding $d_v$-dimensional output values. These are concatenated and once again projected, resulting in the final values, as depicted in Figure~\ref{fig:multi-head-att}.
+
+Multi-head attention allows the model to jointly attend to information from different representation subspaces at different positions. With a single attention head, averaging inhibits this.
+
+\begin{align*}
+    \mathrm{MultiHead}(Q, K, V) &= \mathrm{Concat}(\mathrm{head_1}, ..., \mathrm{head_h})W^O\\
+%    \mathrm{where} \mathrm{head_i} &= \mathrm{Attention}(QW_Q_i^{\dmodel \times d_q}, KW_K_i^{\dmodel \times d_k}, VW^V_i^{\dmodel \times d_v})\\
+    \text{where}~\mathrm{head_i} &= \mathrm{Attention}(QW^Q_i, KW^K_i, VW^V_i)\\
+\end{align*}
+
+Where the projections are parameter matrices $W^Q_i \in \mathbb{R}^{\dmodel \times d_k}$, $W^K_i \in \mathbb{R}^{\dmodel \times d_k}$, $W^V_i \in \mathbb{R}^{\dmodel \times d_v}$ and $W^O \in \mathbb{R}^{hd_v \times \dmodel}$.
+
+
+%find it better (and no more expensive) to have multiple parallel attention layers (each over the full set of positions) with proportionally lower-dimensional keys, values and queries.  We call this "Multi-Head Attention" (Figure~\ref{fig:multi-head-att}).  The keys, values, and queries for each of these parallel attention layers are computed by learned linear transformations of the inputs to the multi-head attention.  We use different linear transformations across different parallel attention layers.  The output of the parallel attention layers are concatenated, and then passed through a final learned linear transformation. 
+
+In this work we employ $h=8$ parallel attention layers, or heads. For each of these we use $d_k=d_v=\dmodel/h=64$.
+Due to the reduced dimension of each head, the total computational cost is similar to that of single-head attention with full dimensionality.
+
+\subsubsection{Applications of Attention in our Model}
+
+The Transformer uses multi-head attention in three different ways: 
+\begin{itemize}
+ \item In "encoder-decoder attention" layers, the queries come from the previous decoder layer, and the memory keys and values come from the output of the encoder.   This allows every position in the decoder to attend over all positions in the input sequence.  This mimics the typical encoder-decoder attention mechanisms in sequence-to-sequence models such as \citep{wu2016google, bahdanau2014neural,JonasFaceNet2017}.
+
+ \item The encoder contains self-attention layers.  In a self-attention layer all of the keys, values and queries come from the same place, in this case, the output of the previous layer in the encoder.   Each position in the encoder can attend to all positions in the previous layer of the encoder.
+
+ \item Similarly, self-attention layers in the decoder allow each position in the decoder to attend to all positions in the decoder up to and including that position.  We need to prevent leftward information flow in the decoder to preserve the auto-regressive property.  We implement this inside of scaled dot-product attention by masking out (setting to $-\infty$) all values in the input of the softmax which correspond to illegal connections.  See Figure~\ref{fig:multi-head-att}.
+
+\end{itemize}
+
+\subsection{Position-wise Feed-Forward Networks}\label{sec:ffn}
+
+In addition to attention sub-layers, each of the layers in our encoder and decoder contains a fully connected feed-forward network, which is applied to each position separately and identically.  This consists of two linear transformations with a ReLU activation in between.
+
+\begin{equation}
+   \mathrm{FFN}(x)=\max(0, xW_1 + b_1) W_2 + b_2
+\end{equation}
+
+While the linear transformations are the same across different positions, they use different parameters from layer to layer. Another way of describing this is as two convolutions with kernel size 1.  The dimensionality of input and output is $\dmodel=512$, and the inner-layer has dimensionality $d_{ff}=2048$.
+
+
+
+%In the appendix, we describe how the position-wise feed-forward network can also be seen as a form of attention.
+
+%from Jakob: The number of operations required for the model to relate signals from two arbitrary input or output positions grows in the distance between positions in input or output, linearly for ConvS2S and logarithmically for ByteNet, making it harder to learn dependencies between these positions \citep{hochreiter2001gradient}. In the transformer this is reduced to a constant number of operations, albeit at the cost of effective resolution caused by averaging attention-weighted positions, an effect we aim to counteract with multi-headed attention.
+
+
+%Figure~\ref{fig:simple-att} presents a simple attention function, $A$, with a single head, that forms the basis of our multi-head attention. $A$ takes a query key vector $\kq$, matrices of memory keys $\km$ and memory values $\vm$ ,and produces a query value vector $\vq$ as 
+%\begin{equation*} \label{eq:attention}
+%    A(\kq, \km, \vm) = {\vm}^T (Softmax(\km \kq).
+%\end{equation*}
+%We linearly transform $\kq,\,\km$, and $\vm$ with learned matrices ${\Wkq \text{,} \, \Wkm}$, and ${\Wvm}$ before calling the attention function, and transform the output query with $\Wvq$ before handing it to the feed forward layer. Each attention layer has it's own set of transformation matrices, which are shared across all query positions. $A$ is applied in parallel for each query position, and is implemented very efficiently as a batch of matrix multiplies. The self-attention and encoder-decoder attention layers use $A$, but with different arguments. For example, in encdoder self-attention, queries in encoder layer $i$ attention to memories in encoder layer $i-1$. To ensure that decoder self-attention layers do not look at future words, we add $- \inf$ to the softmax logits in positions $j+1$ to query length for query position $l$.  
+
+%In simple attention, the query value is a weighted combination of the memory values where the attention weights sum to one. Although this function performs well in practice, the constraint on attention weights can restrict the amount of information that flows from memories to queries because the query cannot focus on multiple memory positions at once, which might be desirable when translating long sequences. \marginpar{@usz, could you think of an example of this ?} We remedy this by maintaining multiple attention heads at each query position that attend to all memory positions in parallel, with a different set of parameters  per attention head $h$. 
+%\marginpar{}
+
+\subsection{Embeddings and Softmax}
+Similarly to other sequence transduction models, we use learned embeddings to convert the input tokens and output tokens to vectors of dimension $\dmodel$.  We also use the usual learned linear transformation and softmax function to convert the decoder output to predicted next-token probabilities.  In our model, we share the same weight matrix between the two embedding layers and the pre-softmax linear transformation, similar to \citep{press2016using}.   In the embedding layers, we multiply those weights by $\sqrt{\dmodel}$.
+
+
+\subsection{Positional Encoding}
+Since our model contains no recurrence and no convolution, in order for the model to make use of the order of the sequence, we must inject some information about the relative or absolute position of the tokens in the sequence.  To this end, we add "positional encodings" to the input embeddings at the bottoms of the encoder and decoder stacks.  The positional encodings have the same dimension $\dmodel$ as the embeddings, so that the two can be summed.   There are many choices of positional encodings, learned and fixed \citep{JonasFaceNet2017}.
+
+In this work, we use sine and cosine functions of different frequencies:
+
+\begin{align*}
+    PE_{(pos,2i)} = sin(pos / 10000^{2i/\dmodel}) \\
+    PE_{(pos,2i+1)} = cos(pos / 10000^{2i/\dmodel})
+\end{align*}
+
+where $pos$ is the position and $i$ is the dimension.  That is, each dimension of the positional encoding corresponds to a sinusoid.  The wavelengths form a geometric progression from $2\pi$ to $10000 \cdot 2\pi$.  We chose this function because we hypothesized it would allow the model to easily learn to attend by relative positions, since for any fixed offset $k$, $PE_{pos+k}$ can be represented as a linear function of $PE_{pos}$.
+
+We also experimented with using learned positional embeddings \citep{JonasFaceNet2017} instead, and found that the two versions produced nearly identical results (see Table~\ref{tab:variations} row (E)).  We chose the sinusoidal version because it may allow the model to extrapolate to sequence lengths longer than the ones encountered during training.

download/20230719163556/output/Attention-Is-All-You-Need/ms.tex

@@ -0,0 +1,413 @@
+\documentclass{article}
+
+% if you need to pass options to natbib, use, e.g.:
+ \PassOptionsToPackage{numbers, compress}{natbib}
+% before loading nips_2017
+%
+% to avoid loading the natbib package, add option nonatbib:
+% \usepackage[nonatbib]{nips_2017}
+
+% Keep final for now to remove annoying line numbers.
+%\usepackage[final]{nips_2017}
+% Keep final for now to remove annoying line numbers.
+
+
+% to compile a camera-ready version, add the [final] option, e.g.:
+\usepackage[final]{nips_2017}
+
+\usepackage[utf8]{inputenc} % allow utf-8 input
+\usepackage[T1]{fontenc}    % use 8-bit T1 fonts
+\usepackage{hyperref}       % hyperlinks
+\usepackage{url}            % simple URL typesetting
+\usepackage{booktabs}       % professional-quality tables
+\usepackage{amsfonts}       % blackboard math symbols
+\usepackage{amsmath}
+\usepackage{nicefrac}       % compact symbols for 1/2, etc.
+\usepackage{microtype}      % microtypography
+\usepackage{subfiles}
+\usepackage{xcolor}
+\usepackage{multirow}
+\usepackage{enumerate}
+\usepackage{subfiles}
+\usepackage{multirow}
+\usepackage{graphicx}
+\usepackage{subfiles}
+
+
+% % Custom Commands:
+% \newcommand\blfootnote[1]{%
+%   \begingroup
+%   \renewcommand\thefootnote{}\footnote{#1}%
+%   \addtocounter{footnote}{-1}%
+%   \endgroup
+% }
+
+\newcommand\todo[1]{\textcolor{red}{[[#1]]}}
+\newcommand\mc[1]{\mathcal{#1}}
+\newcommand*\samethanks[1][\value{footnote}]{\footnotemark[#1]}
+%keys for memory and values. Can be changed if needed
+\newcommand{\kq}{q}
+\newcommand{\km}{k}
+\newcommand{\vq}{o}
+\newcommand{\vm}{m}
+\newcommand{\Wkq}{W_q}
+\newcommand{\Wkm}{W_k}
+\newcommand{\Wvq}{W_o}
+\newcommand{\Wvm}{W_m}
+\newcommand{\dmodel}{d_{\text{model}}}
+\newcommand{\dffn}{d_{\text{ffn}}}
+\newcommand{\dff}{d_{\text{ff}}}
+\newcommand{\mbf}[1]{\mathbf{#1}}
+%\newcommand{\kq}{{q}_k}
+%\newcommand{\km}{{m}_k}
+%\newcommand{\vq}{{q}_v}
+%\newcommand{\vm}{{m}_v}
+%\newcommand{\Wkq}{{W_q}_k}
+%\newcommand{\Wkm}{{W_m}_k}
+%\newcommand{\Wvq}{{W_q}_v}
+%\newcommand{\Wvm}{{W_m}_v}
+\newcommand\concat[3]{\left[#1 \parallel_#3 #2\right]}
+
+\title{Attention Is All You Need}
+
+% The \author macro works with any number of authors. There are two
+% commands used to separate the names and addresses of multiple
+% authors: \And and \AND.
+%
+% Using \And between authors leaves it to LaTeX to determine where to
+% break the lines. Using \AND forces a line break at that point. So,
+% if LaTeX puts 3 of 4 authors names on the first line, and the last
+% on the second line, try using \AND instead of \And before the third
+% author name.
+\author{
+  \AND
+  Ashish Vaswani\thanks{Equal contribution. Listing order is random. Jakob proposed replacing RNNs with self-attention and started the effort to evaluate this idea.
+Ashish, with Illia, designed and implemented the first Transformer models and has been crucially involved in every aspect of this work. Noam proposed scaled dot-product attention, multi-head attention and the parameter-free position representation and became the other person involved in nearly every detail. Niki designed, implemented, tuned and evaluated countless model variants in our original codebase and tensor2tensor. Llion also experimented with novel model variants, was responsible for our initial codebase, and efficient inference and visualizations. Lukasz and Aidan spent countless long days designing various parts of and implementing tensor2tensor, replacing our earlier codebase, greatly improving results and massively accelerating our research.
+}\\
+  Google Brain\\
+  \texttt{[email protected]}\\
+  \And
+  Noam Shazeer\footnotemark[1]\\
+  Google Brain\\
+  \texttt{[email protected]}\\
+  \And
+  Niki Parmar\footnotemark[1]\\
+  Google Research\\
+  \texttt{[email protected]}\\  
+  \And
+  Jakob Uszkoreit\footnotemark[1]\\
+  Google Research\\
+  \texttt{[email protected]}\\
+  \And  
+  Llion Jones\footnotemark[1]\\
+  Google Research\\
+  \texttt{[email protected]}\\   
+  \And
+  Aidan N. Gomez\footnotemark[1] \hspace{1.7mm}\thanks{Work performed while at Google Brain.}\\
+  University of Toronto\\
+  \texttt{[email protected]}
+  \And
+  {\L}ukasz Kaiser\footnotemark[1]\\
+  Google Brain\\
+  \texttt{[email protected]}\\
+  \And
+  Illia Polosukhin\footnotemark[1]\hspace{1.7mm} \thanks{Work performed while at Google Research.}\\
+  \texttt{[email protected]}\\  
+}
+
+\begin{document}
+
+\maketitle
+
+\begin{abstract}
+The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being  more parallelizable and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles, by over 2 BLEU.  On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature. We show that the Transformer generalizes well to other tasks by applying it successfully to English constituency parsing  both with large and limited training data.
+% \blfootnote{Code available at \url{https://github.com/tensorflow/tensor2tensor}}
+
+%TODO(noam): update results for new models.
+
+%llion@: FAIR's paper seems to concentrate solely on the convolutional aspect of their model and have the attention as an after thought almost, this gives us a good opportunity to differentiate ourselves from their paper.
+
+%We are simpler in a number of ways and should have the simplicity as a big selling point:
+%\begin{itemize}
+%\item No convolutions
+%\item No need for such careful initializations and %normalization.
+%\item Simpler non-lineararities, they use the gated linear %units.
+%\item Less layers?
+%\end{itemize}
+%One thing we do more is that we have self attention.
+%Another selling point is the increased interpretability as %shown with the visualizations. Which comes from the %simplicity and use of only attentions.
+\end{abstract}
+
+\section{Introduction}
+
+\input{introduction}
+
+\section{Background}
+
+\input{background}
+
+\section{Model Architecture}
+\input{model_architecture}
+ 
+\section{Why Self-Attention}
+\input{why_self_attention}
+
+\section{Training}
+\input{training}
+ 
+\section{Results} \label{sec:results}
+\input{results}
+
+\section{Conclusion}
+In this work, we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers most commonly used in encoder-decoder architectures with multi-headed self-attention.
+
+For translation tasks, the Transformer can be trained significantly faster than architectures based on recurrent or convolutional layers. On both WMT 2014 English-to-German and WMT 2014 English-to-French translation tasks, we achieve a new state of the art. In the former task our best model outperforms even all previously reported ensembles. %We also provide an indication of the broader applicability of our models through experiments on English constituency parsing.
+
+We are excited about the future of attention-based models and plan to apply them to other tasks. We plan to extend the Transformer to problems involving input and output modalities other than text and to investigate local, restricted attention mechanisms to efficiently handle large inputs and outputs such as images, audio and video.
+Making generation less sequential is another research goals of ours.
+
+The code we used to train and evaluate our models is available at \url{https://github.com/tensorflow/tensor2tensor}.
+
+\paragraph{Acknowledgements} We are grateful to Nal Kalchbrenner and Stephan Gouws for
+their fruitful comments, corrections and inspiration.
+
+\bibliographystyle{plain}
+%\bibliography{deeplearn}
+\begin{thebibliography}{10}
+
+\bibitem{layernorm2016}
+Jimmy~Lei Ba, Jamie~Ryan Kiros, and Geoffrey~E Hinton.
+\newblock Layer normalization.
+\newblock {\em arXiv preprint arXiv:1607.06450}, 2016.
+
+\bibitem{bahdanau2014neural}
+Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Bengio.
+\newblock Neural machine translation by jointly learning to align and
+  translate.
+\newblock {\em CoRR}, abs/1409.0473, 2014.
+
+\bibitem{DBLP:journals/corr/BritzGLL17}
+Denny Britz, Anna Goldie, Minh{-}Thang Luong, and Quoc~V. Le.
+\newblock Massive exploration of neural machine translation architectures.
+\newblock {\em CoRR}, abs/1703.03906, 2017.
+
+\bibitem{cheng2016long}
+Jianpeng Cheng, Li~Dong, and Mirella Lapata.
+\newblock Long short-term memory-networks for machine reading.
+\newblock {\em arXiv preprint arXiv:1601.06733}, 2016.
+
+\bibitem{cho2014learning}
+Kyunghyun Cho, Bart van Merrienboer, Caglar Gulcehre, Fethi Bougares, Holger
+  Schwenk, and Yoshua Bengio.
+\newblock Learning phrase representations using rnn encoder-decoder for
+  statistical machine translation.
+\newblock {\em CoRR}, abs/1406.1078, 2014.
+
+\bibitem{xception2016}
+Francois Chollet.
+\newblock Xception: Deep learning with depthwise separable convolutions.
+\newblock {\em arXiv preprint arXiv:1610.02357}, 2016.
+
+\bibitem{gruEval14}
+Junyoung Chung, {\c{C}}aglar G{\"{u}}l{\c{c}}ehre, Kyunghyun Cho, and Yoshua
+  Bengio.
+\newblock Empirical evaluation of gated recurrent neural networks on sequence
+  modeling.
+\newblock {\em CoRR}, abs/1412.3555, 2014.
+
+\bibitem{dyer-rnng:16}
+Chris Dyer, Adhiguna Kuncoro, Miguel Ballesteros, and Noah~A. Smith.
+\newblock Recurrent neural network grammars.
+\newblock In {\em Proc. of NAACL}, 2016.
+
+\bibitem{JonasFaceNet2017}
+Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, and Yann~N. Dauphin.
+\newblock Convolutional sequence to sequence learning.
+\newblock {\em arXiv preprint arXiv:1705.03122v2}, 2017.
+
+\bibitem{graves2013generating}
+Alex Graves.
+\newblock Generating sequences with recurrent neural networks.
+\newblock {\em arXiv preprint arXiv:1308.0850}, 2013.
+
+\bibitem{he2016deep}
+Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun.
+\newblock Deep residual learning for image recognition.
+\newblock In {\em Proceedings of the IEEE Conference on Computer Vision and
+  Pattern Recognition}, pages 770--778, 2016.
+
+\bibitem{hochreiter2001gradient}
+Sepp Hochreiter, Yoshua Bengio, Paolo Frasconi, and J{\"u}rgen Schmidhuber.
+\newblock Gradient flow in recurrent nets: the difficulty of learning long-term
+  dependencies, 2001.
+
+\bibitem{hochreiter1997}
+Sepp Hochreiter and J{\"u}rgen Schmidhuber.
+\newblock Long short-term memory.
+\newblock {\em Neural computation}, 9(8):1735--1780, 1997.
+
+\bibitem{huang-harper:2009:EMNLP}
+Zhongqiang Huang and Mary Harper.
+\newblock Self-training {PCFG} grammars with latent annotations across
+  languages.
+\newblock In {\em Proceedings of the 2009 Conference on Empirical Methods in
+  Natural Language Processing}, pages 832--841. ACL, August 2009.
+
+\bibitem{jozefowicz2016exploring}
+Rafal Jozefowicz, Oriol Vinyals, Mike Schuster, Noam Shazeer, and Yonghui Wu.
+\newblock Exploring the limits of language modeling.
+\newblock {\em arXiv preprint arXiv:1602.02410}, 2016.
+
+\bibitem{extendedngpu}
+{\L}ukasz Kaiser and Samy Bengio.
+\newblock Can active memory replace attention?
+\newblock In {\em Advances in Neural Information Processing Systems, ({NIPS})},
+  2016.
+
+\bibitem{neural_gpu}
+\L{}ukasz Kaiser and Ilya Sutskever.
+\newblock Neural {GPU}s learn algorithms.
+\newblock In {\em International Conference on Learning Representations
+  ({ICLR})}, 2016.
+
+\bibitem{NalBytenet2017}
+Nal Kalchbrenner, Lasse Espeholt, Karen Simonyan, Aaron van~den Oord, Alex
+  Graves, and Koray Kavukcuoglu.
+\newblock Neural machine translation in linear time.
+\newblock {\em arXiv preprint arXiv:1610.10099v2}, 2017.
+
+\bibitem{structuredAttentionNetworks}
+Yoon Kim, Carl Denton, Luong Hoang, and Alexander~M. Rush.
+\newblock Structured attention networks.
+\newblock In {\em International Conference on Learning Representations}, 2017.
+
+\bibitem{kingma2014adam}
+Diederik Kingma and Jimmy Ba.
+\newblock Adam: A method for stochastic optimization.
+\newblock In {\em ICLR}, 2015.
+
+\bibitem{Kuchaiev2017Factorization}
+Oleksii Kuchaiev and Boris Ginsburg.
+\newblock Factorization tricks for {LSTM} networks.
+\newblock {\em arXiv preprint arXiv:1703.10722}, 2017.
+
+\bibitem{lin2017structured}
+Zhouhan Lin, Minwei Feng, Cicero Nogueira~dos Santos, Mo~Yu, Bing Xiang, Bowen
+  Zhou, and Yoshua Bengio.
+\newblock A structured self-attentive sentence embedding.
+\newblock {\em arXiv preprint arXiv:1703.03130}, 2017.
+
+\bibitem{multiseq2seq}
+Minh-Thang Luong, Quoc~V. Le, Ilya Sutskever, Oriol Vinyals, and Lukasz Kaiser.
+\newblock Multi-task sequence to sequence learning.
+\newblock {\em arXiv preprint arXiv:1511.06114}, 2015.
+
+\bibitem{luong2015effective}
+Minh-Thang Luong, Hieu Pham, and Christopher~D Manning.
+\newblock Effective approaches to attention-based neural machine translation.
+\newblock {\em arXiv preprint arXiv:1508.04025}, 2015.
+
+\bibitem{marcus1993building}
+Mitchell~P Marcus, Mary~Ann Marcinkiewicz, and Beatrice Santorini.
+\newblock Building a large annotated corpus of english: The penn treebank.
+\newblock {\em Computational linguistics}, 19(2):313--330, 1993.
+
+\bibitem{mcclosky-etAl:2006:NAACL}
+David McClosky, Eugene Charniak, and Mark Johnson.
+\newblock Effective self-training for parsing.
+\newblock In {\em Proceedings of the Human Language Technology Conference of
+  the NAACL, Main Conference}, pages 152--159. ACL, June 2006.
+
+\bibitem{decomposableAttnModel}
+Ankur Parikh, Oscar Täckström, Dipanjan Das, and Jakob Uszkoreit.
+\newblock A decomposable attention model.
+\newblock In {\em Empirical Methods in Natural Language Processing}, 2016.
+
+\bibitem{paulus2017deep}
+Romain Paulus, Caiming Xiong, and Richard Socher.
+\newblock A deep reinforced model for abstractive summarization.
+\newblock {\em arXiv preprint arXiv:1705.04304}, 2017.
+
+\bibitem{petrov-EtAl:2006:ACL}
+Slav Petrov, Leon Barrett, Romain Thibaux, and Dan Klein.
+\newblock Learning accurate, compact, and interpretable tree annotation.
+\newblock In {\em Proceedings of the 21st International Conference on
+  Computational Linguistics and 44th Annual Meeting of the ACL}, pages
+  433--440. ACL, July 2006.
+
+\bibitem{press2016using}
+Ofir Press and Lior Wolf.
+\newblock Using the output embedding to improve language models.
+\newblock {\em arXiv preprint arXiv:1608.05859}, 2016.
+
+\bibitem{sennrich2015neural}
+Rico Sennrich, Barry Haddow, and Alexandra Birch.
+\newblock Neural machine translation of rare words with subword units.
+\newblock {\em arXiv preprint arXiv:1508.07909}, 2015.
+
+\bibitem{shazeer2017outrageously}
+Noam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le,
+  Geoffrey Hinton, and Jeff Dean.
+\newblock Outrageously large neural networks: The sparsely-gated
+  mixture-of-experts layer.
+\newblock {\em arXiv preprint arXiv:1701.06538}, 2017.
+
+\bibitem{srivastava2014dropout}
+Nitish Srivastava, Geoffrey~E Hinton, Alex Krizhevsky, Ilya Sutskever, and
+  Ruslan Salakhutdinov.
+\newblock Dropout: a simple way to prevent neural networks from overfitting.
+\newblock {\em Journal of Machine Learning Research}, 15(1):1929--1958, 2014.
+
+\bibitem{sukhbaatar2015}
+Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, and Rob Fergus.
+\newblock End-to-end memory networks.
+\newblock In C.~Cortes, N.~D. Lawrence, D.~D. Lee, M.~Sugiyama, and R.~Garnett,
+  editors, {\em Advances in Neural Information Processing Systems 28}, pages
+  2440--2448. Curran Associates, Inc., 2015.
+
+\bibitem{sutskever14}
+Ilya Sutskever, Oriol Vinyals, and Quoc~VV Le.
+\newblock Sequence to sequence learning with neural networks.
+\newblock In {\em Advances in Neural Information Processing Systems}, pages
+  3104--3112, 2014.
+
+\bibitem{DBLP:journals/corr/SzegedyVISW15}
+Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, and
+  Zbigniew Wojna.
+\newblock Rethinking the inception architecture for computer vision.
+\newblock {\em CoRR}, abs/1512.00567, 2015.
+
+\bibitem{KVparse15}
+{Vinyals {\&} Kaiser}, Koo, Petrov, Sutskever, and Hinton.
+\newblock Grammar as a foreign language.
+\newblock In {\em Advances in Neural Information Processing Systems}, 2015.
+
+\bibitem{wu2016google}
+Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc~V Le, Mohammad Norouzi, Wolfgang
+  Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, et~al.
+\newblock Google's neural machine translation system: Bridging the gap between
+  human and machine translation.
+\newblock {\em arXiv preprint arXiv:1609.08144}, 2016.
+
+\bibitem{DBLP:journals/corr/ZhouCWLX16}
+Jie Zhou, Ying Cao, Xuguang Wang, Peng Li, and Wei Xu.
+\newblock Deep recurrent models with fast-forward connections for neural
+  machine translation.
+\newblock {\em CoRR}, abs/1606.04199, 2016.
+
+\bibitem{zhu-EtAl:2013:ACL}
+Muhua Zhu, Yue Zhang, Wenliang Chen, Min Zhang, and Jingbo Zhu.
+\newblock Fast and accurate shift-reduce constituent parsing.
+\newblock In {\em Proceedings of the 51st Annual Meeting of the ACL (Volume 1:
+  Long Papers)}, pages 434--443. ACL, August 2013.
+
+\end{thebibliography}
+%\newpage
+\input{visualizations}
+%\appendix
+%\newpage
+%\input{parameter_attention}
+
+%\input{sqrt_d_trick}
+
+\end{document}

download/20230719163556/output/Attention-Is-All-You-Need/nips_2017.sty

@@ -0,0 +1,338 @@
+% partial rewrite of the LaTeX2e package for submissions to the
+% Conference on Neural Information Processing Systems (NIPS):
+%
+% - uses more LaTeX conventions
+% - line numbers at submission time replaced with aligned numbers from
+%   lineno package
+% - \nipsfinalcopy replaced with [final] package option
+% - automatically loads times package for authors
+% - loads natbib automatically; this can be suppressed with the
+%   [nonatbib] package option
+% - adds foot line to first page identifying the conference
+%
+% Roman Garnett ([email protected]) and the many authors of
+% nips15submit_e.sty, including MK and drstrip@sandia
+%
+% last revision: March 2017
+
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesPackage{nips_2017}[2017/03/20 NIPS 2017 submission/camera-ready style file]
+
+% declare final option, which creates camera-ready copy
+\newif\if@nipsfinal\@nipsfinalfalse
+\DeclareOption{final}{
+  \@nipsfinaltrue
+}
+
+% declare nonatbib option, which does not load natbib in case of
+% package clash (users can pass options to natbib via
+% \PassOptionsToPackage)
+\newif\if@natbib\@natbibtrue
+\DeclareOption{nonatbib}{
+  \@natbibfalse
+}
+
+\ProcessOptions\relax
+
+% fonts
+\renewcommand{\rmdefault}{ptm}
+\renewcommand{\sfdefault}{phv}
+
+% change this every year for notice string at bottom
+\newcommand{\@nipsordinal}{31st}
+\newcommand{\@nipsyear}{2017}
+\newcommand{\@nipslocation}{Long Beach, CA, USA}
+
+% handle tweaks for camera-ready copy vs. submission copy
+\if@nipsfinal
+  \newcommand{\@noticestring}{%
+    \@nipsordinal\/ Conference on Neural Information Processing Systems
+    (NIPS \@nipsyear), \@nipslocation.%
+  }
+\else
+  \newcommand{\@noticestring}{%
+    Submitted to \@nipsordinal\/ Conference on Neural Information
+    Processing Systems (NIPS \@nipsyear). Do not distribute.%
+  }
+
+  % line numbers for submission
+  \RequirePackage{lineno}
+  \linenumbers
+
+  % fix incompatibilities between lineno and amsmath, if required, by
+  % transparently wrapping linenomath environments around amsmath
+  % environments
+  \AtBeginDocument{%
+    \@ifpackageloaded{amsmath}{%
+      \newcommand*\patchAmsMathEnvironmentForLineno[1]{%
+        \expandafter\let\csname old#1\expandafter\endcsname\csname #1\endcsname
+        \expandafter\let\csname oldend#1\expandafter\endcsname\csname end#1\endcsname
+        \renewenvironment{#1}%
+          {\linenomath\csname old#1\endcsname}%
+          {\csname oldend#1\endcsname\endlinenomath}%
+      }%
+      \newcommand*\patchBothAmsMathEnvironmentsForLineno[1]{%
+        \patchAmsMathEnvironmentForLineno{#1}%
+        \patchAmsMathEnvironmentForLineno{#1*}%
+      }%
+      \patchBothAmsMathEnvironmentsForLineno{equation}%
+      \patchBothAmsMathEnvironmentsForLineno{align}%
+      \patchBothAmsMathEnvironmentsForLineno{flalign}%
+      \patchBothAmsMathEnvironmentsForLineno{alignat}%
+      \patchBothAmsMathEnvironmentsForLineno{gather}%
+      \patchBothAmsMathEnvironmentsForLineno{multline}%
+    }{}
+  }
+\fi
+
+% load natbib unless told otherwise
+\if@natbib
+  \RequirePackage{natbib}
+\fi
+
+% set page geometry
+\usepackage[verbose=true,letterpaper]{geometry}
+\AtBeginDocument{
+  \newgeometry{
+    textheight=9in,
+    textwidth=5.5in,
+    top=1in,
+    headheight=12pt,
+    headsep=25pt,
+    footskip=30pt
+  }
+  \@ifpackageloaded{fullpage}
+    {\PackageWarning{nips_2016}{fullpage package not allowed! Overwriting formatting.}}
+    {}
+}
+
+\widowpenalty=10000
+\clubpenalty=10000
+\flushbottom
+\sloppy
+
+% font sizes with reduced leading
+\renewcommand{\normalsize}{%
+  \@setfontsize\normalsize\@xpt\@xipt
+  \abovedisplayskip      7\p@ \@plus 2\p@ \@minus 5\p@
+  \abovedisplayshortskip \z@ \@plus 3\p@
+  \belowdisplayskip      \abovedisplayskip
+  \belowdisplayshortskip 4\p@ \@plus 3\p@ \@minus 3\p@
+}
+\normalsize
+\renewcommand{\small}{%
+  \@setfontsize\small\@ixpt\@xpt
+  \abovedisplayskip      6\p@ \@plus 1.5\p@ \@minus 4\p@
+  \abovedisplayshortskip \z@  \@plus 2\p@
+  \belowdisplayskip      \abovedisplayskip
+  \belowdisplayshortskip 3\p@ \@plus 2\p@   \@minus 2\p@
+}
+\renewcommand{\footnotesize}{\@setfontsize\footnotesize\@ixpt\@xpt}
+\renewcommand{\scriptsize}{\@setfontsize\scriptsize\@viipt\@viiipt}
+\renewcommand{\tiny}{\@setfontsize\tiny\@vipt\@viipt}
+\renewcommand{\large}{\@setfontsize\large\@xiipt{14}}
+\renewcommand{\Large}{\@setfontsize\Large\@xivpt{16}}
+\renewcommand{\LARGE}{\@setfontsize\LARGE\@xviipt{20}}
+\renewcommand{\huge}{\@setfontsize\huge\@xxpt{23}}
+\renewcommand{\Huge}{\@setfontsize\Huge\@xxvpt{28}}
+
+% sections with less space
+\providecommand{\section}{}
+\renewcommand{\section}{%
+  \@startsection{section}{1}{\z@}%
+                {-2.0ex \@plus -0.5ex \@minus -0.2ex}%
+                { 1.5ex \@plus  0.3ex \@minus  0.2ex}%
+                {\large\bf\raggedright}%
+}
+\providecommand{\subsection}{}
+\renewcommand{\subsection}{%
+  \@startsection{subsection}{2}{\z@}%
+                {-1.8ex \@plus -0.5ex \@minus -0.2ex}%
+                { 0.8ex \@plus  0.2ex}%
+                {\normalsize\bf\raggedright}%
+}
+\providecommand{\subsubsection}{}
+\renewcommand{\subsubsection}{%
+  \@startsection{subsubsection}{3}{\z@}%
+                {-1.5ex \@plus -0.5ex \@minus -0.2ex}%
+                { 0.5ex \@plus  0.2ex}%
+                {\normalsize\bf\raggedright}%
+}
+\providecommand{\paragraph}{}
+\renewcommand{\paragraph}{%
+  \@startsection{paragraph}{4}{\z@}%
+                {1.5ex \@plus 0.5ex \@minus 0.2ex}%
+                {-1em}%
+                {\normalsize\bf}%
+}
+\providecommand{\subparagraph}{}
+\renewcommand{\subparagraph}{%
+  \@startsection{subparagraph}{5}{\z@}%
+                {1.5ex \@plus 0.5ex \@minus 0.2ex}%
+                {-1em}%
+                {\normalsize\bf}%
+}
+\providecommand{\subsubsubsection}{}
+\renewcommand{\subsubsubsection}{%
+  \vskip5pt{\noindent\normalsize\rm\raggedright}%
+}
+
+% float placement
+\renewcommand{\topfraction      }{0.85}
+\renewcommand{\bottomfraction   }{0.4}
+\renewcommand{\textfraction     }{0.1}
+\renewcommand{\floatpagefraction}{0.7}
+
+\newlength{\@nipsabovecaptionskip}\setlength{\@nipsabovecaptionskip}{7\p@}
+\newlength{\@nipsbelowcaptionskip}\setlength{\@nipsbelowcaptionskip}{\z@}
+
+\setlength{\abovecaptionskip}{\@nipsabovecaptionskip}
+\setlength{\belowcaptionskip}{\@nipsbelowcaptionskip}
+
+% swap above/belowcaptionskip lengths for tables
+\renewenvironment{table}
+  {\setlength{\abovecaptionskip}{\@nipsbelowcaptionskip}%
+   \setlength{\belowcaptionskip}{\@nipsabovecaptionskip}%
+   \@float{table}}
+  {\end@float}
+
+% footnote formatting
+\setlength{\footnotesep }{6.65\p@}
+\setlength{\skip\footins}{9\p@ \@plus 4\p@ \@minus 2\p@}
+\renewcommand{\footnoterule}{\kern-3\p@ \hrule width 12pc \kern 2.6\p@}
+\setcounter{footnote}{0}
+
+% paragraph formatting
+\setlength{\parindent}{\z@}
+\setlength{\parskip  }{5.5\p@}
+
+% list formatting
+\setlength{\topsep       }{4\p@ \@plus 1\p@   \@minus 2\p@}
+\setlength{\partopsep    }{1\p@ \@plus 0.5\p@ \@minus 0.5\p@}
+\setlength{\itemsep      }{2\p@ \@plus 1\p@   \@minus 0.5\p@}
+\setlength{\parsep       }{2\p@ \@plus 1\p@   \@minus 0.5\p@}
+\setlength{\leftmargin   }{3pc}
+\setlength{\leftmargini  }{\leftmargin}
+\setlength{\leftmarginii }{2em}
+\setlength{\leftmarginiii}{1.5em}
+\setlength{\leftmarginiv }{1.0em}
+\setlength{\leftmarginv  }{0.5em}
+\def\@listi  {\leftmargin\leftmargini}
+\def\@listii {\leftmargin\leftmarginii
+              \labelwidth\leftmarginii
+              \advance\labelwidth-\labelsep
+              \topsep  2\p@ \@plus 1\p@    \@minus 0.5\p@
+              \parsep  1\p@ \@plus 0.5\p@ \@minus 0.5\p@
+              \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+              \labelwidth\leftmarginiii
+              \advance\labelwidth-\labelsep
+              \topsep    1\p@ \@plus 0.5\p@ \@minus 0.5\p@
+              \parsep    \z@
+              \partopsep 0.5\p@ \@plus 0\p@ \@minus 0.5\p@
+              \itemsep \topsep}
+\def\@listiv {\leftmargin\leftmarginiv
+              \labelwidth\leftmarginiv
+              \advance\labelwidth-\labelsep}
+\def\@listv  {\leftmargin\leftmarginv
+              \labelwidth\leftmarginv
+              \advance\labelwidth-\labelsep}
+\def\@listvi {\leftmargin\leftmarginvi
+              \labelwidth\leftmarginvi
+              \advance\labelwidth-\labelsep}
+
+% create title
+\providecommand{\maketitle}{}
+\renewcommand{\maketitle}{%
+  \par
+  \begingroup
+    \renewcommand{\thefootnote}{\fnsymbol{footnote}}
+    % for perfect author name centering
+    \renewcommand{\@makefnmark}{\hbox to \z@{$^{\@thefnmark}$\hss}}
+    % The footnote-mark was overlapping the footnote-text,
+    % added the following to fix this problem               (MK)
+    \long\def\@makefntext##1{%
+      \parindent 1em\noindent
+      \hbox to 1.8em{\hss $\m@th ^{\@thefnmark}$}##1
+    }
+    \thispagestyle{empty}
+    \@maketitle
+    \@thanks
+    \@notice
+  \endgroup
+  \let\maketitle\relax
+  \let\thanks\relax
+}
+
+% rules for title box at top of first page
+\newcommand{\@toptitlebar}{
+  \hrule height 4\p@
+  \vskip 0.25in
+  \vskip -\parskip%
+}
+\newcommand{\@bottomtitlebar}{
+  \vskip 0.29in
+  \vskip -\parskip
+  \hrule height 1\p@
+  \vskip 0.09in%
+}
+
+% create title (includes both anonymized and non-anonymized versions)
+\providecommand{\@maketitle}{}
+\renewcommand{\@maketitle}{%
+  \vbox{%
+    \hsize\textwidth
+    \linewidth\hsize
+    \vskip 0.1in
+    \@toptitlebar
+    \centering
+    {\LARGE\bf \@title\par}
+    \@bottomtitlebar
+    \if@nipsfinal
+      \def\And{%
+        \end{tabular}\hfil\linebreak[0]\hfil%
+        \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces%
+      }
+      \def\AND{%
+        \end{tabular}\hfil\linebreak[4]\hfil%
+        \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\ignorespaces%
+      }
+      \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}\@author\end{tabular}%
+    \else
+      \begin{tabular}[t]{c}\bf\rule{\z@}{24\p@}
+        Anonymous Author(s) \\
+        Affiliation \\
+        Address \\
+        \texttt{email} \\
+      \end{tabular}%
+    \fi
+    \vskip 0.3in \@minus 0.1in
+  }
+}
+
+% add conference notice to bottom of first page
+\newcommand{\ftype@noticebox}{8}
+\newcommand{\@notice}{%
+  % give a bit of extra room back to authors on first page
+  \enlargethispage{2\baselineskip}%
+  \@float{noticebox}[b]%
+    \footnotesize\@noticestring%
+  \end@float%
+}
+
+% abstract styling
+\renewenvironment{abstract}%
+{%
+  \vskip 0.075in%
+  \centerline%
+  {\large\bf Abstract}%
+  \vspace{0.5ex}%
+  \begin{quote}%
+}
+{
+  \par%
+  \end{quote}%
+  \vskip 1ex%
+}
+
+\endinput

download/20230719163556/output/Attention-Is-All-You-Need/parameter_attention.tex

@@ -0,0 +1,45 @@
+\pagebreak
+\section*{Two Feed-Forward Layers = Attention over Parameters}\label{sec:parameter_attention}
+
+In addition to attention layers, our model contains position-wise feed-forward networks (Section \ref{sec:ffn}), which consist of two linear transformations with a ReLU activation in between.  In fact, these networks too can be seen as a form of attention.  Compare the formula for such a network with the formula for a simple dot-product attention layer (biases and scaling factors omitted):
+
+\begin{align*}
+    FFN(x, W_1, W_2) = ReLU(xW_1)W_2 \\
+    A(q, K, V) = Softmax(qK^T)V
+\end{align*}
+
+Based on the similarity of these formulae, the two-layer feed-forward network can be seen as a kind of attention, where the keys and values are the rows of the trainable parameter matrices $W_1$ and $W_2$, and where we use ReLU instead of Softmax in the compatibility function.
+
+%the compatablity function is $compat(q, k_i) = ReLU(q \cdot k_i)$ instead of $Softmax(qK_T)_i$.
+
+Given this similarity, we experimented with replacing the position-wise feed-forward networks with attention layers similar to the ones we use everywhere else our model. The multi-head-attention-over-parameters sublayer is identical to the multi-head attention described in \ref{sec:multihead}, except that the "keys" and "values" inputs to each attention head are trainable model parameters, as opposed to being linear projections of a previous layer.  These parameters are scaled up by a factor of $\sqrt{d_{model}}$ in order to be more similar to activations.
+
+In our first experiment, we replaced each position-wise feed-forward network with a multi-head-attention-over-parameters sublayer with $h_p=8$ heads, key-dimensionality $d_{pk}=64$, and value-dimensionality $d_{pv}=64$, using $n_p=1536$ key-value pairs for each attention head.  The sublayer has a total of $2097152$ parameters, including the parameters in the query projection and the output projection.  This matches the number of parameters in the position-wise feed-forward network that we replaced.  While the theoretical amount of computation is also the same, in practice, the attention version caused the step times to be about 30\% longer.
+
+In our second experiment, we used $h_p=8$ heads, and $n_p=512$ key-value pairs for each attention head, again matching the total number of parameters in the base model.
+
+Results for the first experiment were slightly worse than for the base model, and results for the second experiment were slightly better, see Table~\ref{tab:parameter_attention}.
+
+\begin{table}[h]
+\caption{Replacing the position-wise feed-forward networks with multihead-attention-over-parameters produces similar results to the base model.  All metrics are on the English-to-German translation development set, newstest2013.}
+\label{tab:parameter_attention}
+\begin{center}
+\vspace{-2mm}
+%\scalebox{1.0}{
+\begin{tabular}{c|cccccc|cccc}
+\hline\rule{0pt}{2.0ex}
+ & \multirow{2}{*}{$\dmodel$} & \multirow{2}{*}{$\dff$} &
+\multirow{2}{*}{$h_p$} & \multirow{2}{*}{$d_{pk}$} & \multirow{2}{*}{$d_{pv}$} &
+ \multirow{2}{*}{$n_p$} &
+ PPL & BLEU & params & training\\
+ & & & & & &  & (dev) & (dev) & $\times10^6$ & time \\
+\hline\rule{0pt}{2.0ex}
+base & 512 & 2048 & & & & & 4.92 & 25.8 & 65 & 12 hours\\
+\hline\rule{0pt}{2.0ex}
+AOP$_1$ & 512 & & 8 & 64 & 64 & 1536 & 4.92& 25.5  & 65 & 16 hours\\
+AOP$_2$ & 512 & & 16 & 64 & 64 & 512 & \textbf{4.86} & \textbf{25.9}  & 65 & 16 hours \\
+\hline
+\end{tabular}
+%}
+\end{center}
+\end{table}

download/20230719163556/output/Attention-Is-All-You-Need/results.tex

@@ -0,0 +1,166 @@
+\subsection{Machine Translation}
+\begin{table}[t]
+\begin{center}
+\caption{The Transformer achieves better BLEU scores than previous state-of-the-art models on the English-to-German and English-to-French newstest2014 tests at a fraction of the training cost.  }
+\label{tab:wmt-results}
+\vspace{-2mm}
+%\scalebox{1.0}{
+\begin{tabular}{lccccc}
+\toprule
+\multirow{2}{*}{\vspace{-2mm}Model} & \multicolumn{2}{c}{BLEU} & & \multicolumn{2}{c}{Training Cost (FLOPs)} \\
+\cmidrule{2-3} \cmidrule{5-6} 
+& EN-DE & EN-FR & & EN-DE & EN-FR \\ 
+\hline
+ByteNet \citep{NalBytenet2017} & 23.75 & & & &\\
+Deep-Att + PosUnk \citep{DBLP:journals/corr/ZhouCWLX16} & & 39.2 & & & $1.0\cdot10^{20}$ \\
+GNMT + RL \citep{wu2016google} & 24.6 & 39.92 & & $2.3\cdot10^{19}$  & $1.4\cdot10^{20}$\\
+ConvS2S \citep{JonasFaceNet2017} & 25.16 & 40.46 & & $9.6\cdot10^{18}$ & $1.5\cdot10^{20}$\\
+MoE \citep{shazeer2017outrageously} & 26.03 & 40.56 & & $2.0\cdot10^{19}$ & $1.2\cdot10^{20}$ \\
+\hline
+\rule{0pt}{2.0ex}Deep-Att + PosUnk Ensemble \citep{DBLP:journals/corr/ZhouCWLX16} & & 40.4 & & &
+ $8.0\cdot10^{20}$ \\
+GNMT + RL Ensemble \citep{wu2016google} & 26.30 & 41.16 & & $1.8\cdot10^{20}$  & $1.1\cdot10^{21}$\\
+ConvS2S Ensemble \citep{JonasFaceNet2017} & 26.36 & \textbf{41.29} & & $7.7\cdot10^{19}$ & $1.2\cdot10^{21}$\\
+\specialrule{1pt}{-1pt}{0pt}
+\rule{0pt}{2.2ex}Transformer (base model) & 27.3 & 38.1 & & \multicolumn{2}{c}{\boldmath$3.3\cdot10^{18}$}\\
+Transformer (big) & \textbf{28.4} & \textbf{41.8} & & \multicolumn{2}{c}{$2.3\cdot10^{19}$} \\
+%\hline
+%\specialrule{1pt}{-1pt}{0pt}
+%\rule{0pt}{2.0ex}
+\bottomrule
+\end{tabular}
+%}
+\end{center}
+\end{table}
+
+
+On the WMT 2014 English-to-German translation task, the big transformer model (Transformer (big) in Table~\ref{tab:wmt-results}) outperforms the best previously reported models (including ensembles) by more than $2.0$ BLEU, establishing a new state-of-the-art BLEU score of $28.4$.  The configuration of this model is listed in the bottom line of Table~\ref{tab:variations}.  Training took $3.5$ days on $8$ P100 GPUs.  Even our base model surpasses all previously published models and ensembles, at a fraction of the training cost of any of the competitive models.
+
+On the WMT 2014 English-to-French translation task, our big model achieves a BLEU score of $41.0$, outperforming all of the previously published single models, at less than $1/4$ the training cost of the previous state-of-the-art model. The Transformer (big) model trained for English-to-French used dropout rate $P_{drop}=0.1$, instead of $0.3$.
+
+For the base models, we used a single model obtained by averaging the last 5 checkpoints, which were written at 10-minute intervals.  For the big models, we averaged the last 20 checkpoints. We used beam search with a beam size of $4$ and length penalty $\alpha=0.6$ \citep{wu2016google}.  These hyperparameters were chosen after experimentation on the development set.  We set the maximum output length during inference to input length + $50$, but terminate early when possible \citep{wu2016google}.
+
+Table \ref{tab:wmt-results} summarizes our results and compares our translation quality and training costs to other model architectures from the literature.  We estimate the number of floating point operations used to train a model by multiplying the training time, the number of GPUs used, and an estimate of the sustained single-precision floating-point capacity of each GPU \footnote{We used values of 2.8, 3.7, 6.0 and 9.5 TFLOPS for K80, K40, M40 and P100, respectively.}.
+%where we compare against the leading machine translation results in the literature. Even our smaller model, with number of parameters comparable to ConvS2S, outperforms all existing single models, and achieves results close to the best ensemble model.
+
+\subsection{Model Variations}
+
+\begin{table}[t]
+\caption{Variations on the Transformer architecture. Unlisted values are identical to those of the base model.  All metrics are on the English-to-German translation development set, newstest2013.  Listed perplexities are per-wordpiece, according to our byte-pair encoding, and should not be compared to per-word perplexities.}
+\label{tab:variations}
+\begin{center}
+\vspace{-2mm}
+%\scalebox{1.0}{
+\begin{tabular}{c|ccccccccc|ccc}
+\hline\rule{0pt}{2.0ex}
+ & \multirow{2}{*}{$N$} & \multirow{2}{*}{$\dmodel$} &
+\multirow{2}{*}{$\dff$} & \multirow{2}{*}{$h$} & 
+\multirow{2}{*}{$d_k$} & \multirow{2}{*}{$d_v$} & 
+\multirow{2}{*}{$P_{drop}$} & \multirow{2}{*}{$\epsilon_{ls}$} &
+train & PPL & BLEU & params \\
+ & & & & & & & & & steps & (dev) & (dev) & $\times10^6$ \\
+% & & & & & & & & & & & & \\
+\hline\rule{0pt}{2.0ex}
+base & 6 & 512 & 2048 & 8 & 64 & 64 & 0.1 & 0.1 & 100K & 4.92 & 25.8 & 65 \\
+\hline\rule{0pt}{2.0ex}
+\multirow{4}{*}{(A)}
+& & & & 1 & 512 & 512 & & & & 5.29 & 24.9 &  \\
+& & & & 4 & 128 & 128 & & & & 5.00 & 25.5 &  \\
+& & & & 16 & 32 & 32 & & & & 4.91 & 25.8 &  \\
+& & & & 32 & 16 & 16 & & & & 5.01 & 25.4 &  \\
+\hline\rule{0pt}{2.0ex}
+\multirow{2}{*}{(B)}
+& & & & & 16 & & & & & 5.16 & 25.1 & 58 \\
+& & & & & 32 & & & & & 5.01 & 25.4 & 60 \\
+\hline\rule{0pt}{2.0ex}
+\multirow{7}{*}{(C)}
+& 2 & & & & & & & &            & 6.11 & 23.7 & 36 \\
+& 4 & & & & & & & &            & 5.19 & 25.3 & 50 \\
+& 8 & & & & & & & &            & 4.88 & 25.5 & 80 \\
+& & 256 & & & 32 & 32 & & &    & 5.75 & 24.5 & 28 \\
+& & 1024 & & & 128 & 128 & & & & 4.66 & 26.0 & 168 \\
+& & & 1024 & & & & & &         & 5.12 & 25.4 & 53 \\
+& & & 4096 & & & & & &         & 4.75 & 26.2 & 90 \\
+\hline\rule{0pt}{2.0ex}
+\multirow{4}{*}{(D)}
+& & & & & & & 0.0 & & & 5.77 & 24.6 &  \\
+& & & & & & & 0.2 & & & 4.95 & 25.5 &  \\
+& & & & & & & & 0.0 & & 4.67 & 25.3 &  \\
+& & & & & & & & 0.2 & & 5.47 & 25.7 &  \\
+\hline\rule{0pt}{2.0ex}
+(E) & & \multicolumn{7}{c}{positional embedding instead of sinusoids} & & 4.92 & 25.7 & \\
+\hline\rule{0pt}{2.0ex}
+big & 6 & 1024 & 4096 & 16 & & & 0.3 & & 300K & \textbf{4.33} & \textbf{26.4} & 213 \\
+\hline
+\end{tabular}
+%}
+\end{center}
+\end{table}
+
+
+%Table \ref{tab:ende-results}. Our base model for this task uses 6 attention layers, 512 hidden dim, 2048 filter dim, 8 attention heads with both attention and symbol dropout of 0.2 and 0.1 respectively. Increasing the filter size of our feed forward component to 8192 increases the BLEU score on En $\to$ De by $?$. For both the models, we use beam search decoding of size $?$ and length penalty with an alpha of $?$ \cite? \todo{Update results}
+
+To evaluate the importance of different components of the Transformer, we varied our base model in different ways, measuring the change in performance on English-to-German translation on the development set, newstest2013. We used beam search as described in the previous section, but no checkpoint averaging.  We present these results in Table~\ref{tab:variations}.  
+
+In Table~\ref{tab:variations} rows (A), we vary the number of attention heads and the attention key and value dimensions, keeping the amount of computation constant, as described in Section \ref{sec:multihead}. While single-head attention is 0.9 BLEU worse than the best setting, quality also drops off with too many heads.
+
+In Table~\ref{tab:variations} rows (B), we observe that reducing the attention key size $d_k$ hurts model quality. This suggests that determining compatibility is not easy and that a more sophisticated compatibility function than dot product may be beneficial. We further observe in rows (C) and (D) that, as expected, bigger models are better, and dropout is very helpful in avoiding over-fitting.  In row (E) we replace our sinusoidal positional encoding with learned positional embeddings \citep{JonasFaceNet2017}, and observe nearly identical results to the base model.
+
+%To evaluate the importance of different components of the Transformer, we use our base model to ablate on a single hyperparameter at each time and measure the change in performance on English$\to$German translation. Our variations in Table~\ref{tab:variations} show that the number of attention layers and attention heads is the most important architecture hyperparamter However, the we do not see performance gains beyond 6 layers, suggesting that we either don't have enough data to train a large model or we need to turn up regularization. We leave this exploration for future work. Among our regularizers, attention dropout has the most significant impact on performance.
+
+
+%Increasing the width of our feed forward component helps both on log ppl and Accuracy \marginpar{Intuition?}
+%Using dropout to regularize our models helps to prevent overfitting
+
+\subsection{English Constituency Parsing}
+
+\begin{table}[t]
+\begin{center}
+\caption{The Transformer generalizes well to English constituency parsing (Results are on Section 23 of WSJ)}
+\label{tab:parsing-results}
+\vspace{-2mm}
+%\scalebox{1.0}{
+\begin{tabular}{c|c|c}
+\hline
+{\bf Parser}  & {\bf Training} & {\bf WSJ 23 F1} \\ \hline
+Vinyals \& Kaiser el al. (2014) \cite{KVparse15}
+  & WSJ only, discriminative & 88.3 \\
+Petrov et al. (2006) \cite{petrov-EtAl:2006:ACL}
+  & WSJ only, discriminative & 90.4 \\
+Zhu et al. (2013) \cite{zhu-EtAl:2013:ACL}
+  & WSJ only, discriminative & 90.4   \\
+Dyer et al. (2016) \cite{dyer-rnng:16}
+  & WSJ only, discriminative & 91.7   \\
+\specialrule{1pt}{-1pt}{0pt}
+Transformer (4 layers)  &  WSJ only, discriminative & 91.3 \\
+\specialrule{1pt}{-1pt}{0pt}   
+Zhu et al. (2013) \cite{zhu-EtAl:2013:ACL}
+  & semi-supervised & 91.3 \\
+Huang \& Harper (2009) \cite{huang-harper:2009:EMNLP}
+  & semi-supervised & 91.3 \\
+McClosky et al. (2006) \cite{mcclosky-etAl:2006:NAACL}
+  & semi-supervised & 92.1 \\
+Vinyals \& Kaiser el al. (2014) \cite{KVparse15}
+  & semi-supervised & 92.1 \\
+\specialrule{1pt}{-1pt}{0pt}
+Transformer (4 layers)  & semi-supervised & 92.7 \\
+\specialrule{1pt}{-1pt}{0pt}   
+Luong et al. (2015) \cite{multiseq2seq}
+  & multi-task & 93.0   \\
+Dyer et al. (2016) \cite{dyer-rnng:16}
+  & generative & 93.3   \\
+\hline
+\end{tabular}
+\end{center}
+\end{table}
+
+To evaluate if the Transformer can generalize to other tasks we performed experiments on English constituency parsing. This task presents specific challenges: the output is subject to strong structural constraints and is significantly longer than the input.
+Furthermore, RNN sequence-to-sequence models have not been able to attain state-of-the-art results in small-data regimes \cite{KVparse15}.
+
+We trained a 4-layer transformer with $d_{model} = 1024$ on the Wall Street Journal (WSJ) portion of the Penn Treebank \citep{marcus1993building}, about 40K training sentences. We also trained it in a semi-supervised setting, using the larger high-confidence and BerkleyParser corpora from with approximately 17M sentences \citep{KVparse15}. We used a vocabulary of 16K tokens for the WSJ only setting and a vocabulary of 32K tokens for the semi-supervised setting.
+
+We performed only a small number of experiments to select the dropout, both attention and residual (section~\ref{sec:reg}), learning rates and beam size on the Section 22 development set, all other parameters remained unchanged from the English-to-German base translation model. During inference, we increased the maximum output length to input length + $300$. We used a beam size of $21$ and $\alpha=0.3$ for both WSJ only and the semi-supervised setting.
+
+Our results in Table~\ref{tab:parsing-results} show that despite the lack of task-specific tuning our model performs surprisingly well, yielding better results than all previously reported models with the exception of the Recurrent Neural Network Grammar \cite{dyer-rnng:16}.
+
+In contrast to RNN sequence-to-sequence models \citep{KVparse15}, the Transformer outperforms the BerkeleyParser \cite{petrov-EtAl:2006:ACL} even when training only on the WSJ training set of 40K sentences.

download/20230719163556/output/Attention-Is-All-You-Need/sqrt_d_trick.tex

@@ -0,0 +1,28 @@
+\section*{Justfication of the Scaling Factor in Dot-product Attention}
+
+In Section~\ref{sec:scaled-dot-prod}, we introduced Scaled dot-product attention, where we scale down the dot products by $\sqrt{d_k}$.   In this section, we will give a rough justification of this scaling factor.  If we assume that $q$ and $k$ are $d_k$-dimensional vectors whose components are independent random variables with mean $0$ and variance $1$, then their dot product, $q \cdot k = \sum_{i=1}^{d_k} u_iv_i$, has mean $0$ and variance $d_k$.  Since we would prefer these values to have variance $1$, we divide by $\sqrt{d_k}$.  
+
+
+
+%For any two $d_k$-dimension vectors $\vec{u}$ and $\vec{v}$, whose dimensions are independent, the mean and variance of the dot product will be the summation of the product of means and variances over the dimensions, that is, $E[<\vec{u},\vec{v}>] = \sum_{i=1}^{d_k} E[u_i]E[v_i]$, and $E[(<\vec{u},\vec{v}>-E[<\vec{u},\vec{v}>])^2] = \sum_{i=1}^{d_k} E[({u_i}-E[u_i])^2] E[({v_i}-E[v_i])^2]$. Layer norm encourages the mean and variance of each dimension to be $0$ and $1$ respectively, resultig in the dot product having mean $0$ and $d_k$ respectively. Therefore, scaling by $\sqrt{d_k}$ encourages the logits to be normalized as well. 
+
+\iffalse
+
+In this section, we will give a rough justification of this scaling factor, that is, we will show that for any two vectors, $\vec{u}$ and $\vec{v}$, whose variance and mean are $1$ and $0$ respectively, the variance and the mean of the dot product are $d_k$ and $0$ respectively. Therefore, dividing by $\sqrt{d_k}$ ensures that each component of the attention logits are normalized. The repeated layer norms at each transformer layer encourage $\vec{u}$ and $\vec{v}$ to be normalized. 
+
+
+\begin{align*}
+    E[<\vec{u},\vec{v}>] & =  \sum_k E[u_i v_i] &\text{By linearity of expectation} \\
+    & =\sum_k E[u_i]E[v_i] & \text{Assuming independence} \\
+    & = 0
+\end{align*}
+
+\begin{align*}
+    E[(<\vec{u},\vec{v}>-E[<\vec{u},\vec{v}>])^2]  & = E[(<\vec{u},\vec{v}>)^2] - E[<\vec{u},\vec{v}>]^2 \\
+    & = E[(<\vec{u},\vec{v}>)^2] \\
+    & =  \sum_k E[{u_i}^2] E[{v_i}^2] &\text{By linearity of expectation and indepedence} \\
+    & = d_k
+\end{align*}
+
+
+\fi

download/20230719163556/output/Attention-Is-All-You-Need/training.tex

@@ -0,0 +1,42 @@
+This section describes the training regime for our models. 
+
+%In order to speed up experimentation, our ablations are performed relative to a smaller base model described in detail in Section \ref{sec:results}.
+
+\subsection{Training Data and Batching}
+We trained on the standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs.  Sentences were encoded using byte-pair encoding \citep{DBLP:journals/corr/BritzGLL17}, which has a shared source-target vocabulary of about 37000 tokens. For English-French, we used the significantly larger WMT 2014 English-French dataset consisting of 36M sentences and split tokens into a 32000 word-piece vocabulary \citep{wu2016google}.  Sentence pairs were batched together by approximate sequence length.  Each training batch contained a set of sentence pairs containing approximately 25000 source tokens and 25000 target tokens.  
+
+\subsection{Hardware and Schedule}
+
+We trained our models on one machine with 8 NVIDIA P100 GPUs.  For our base models using the hyperparameters described throughout the paper, each training step took about 0.4 seconds.  We trained the base models for a total of 100,000 steps or 12 hours.  For our big models,(described on the bottom line of table \ref{tab:variations}), step time was 1.0 seconds.  The big models were trained for 300,000 steps (3.5 days).
+
+\subsection{Optimizer} We used the Adam optimizer~\citep{kingma2014adam} with $\beta_1=0.9$, $\beta_2=0.98$ and $\epsilon=10^{-9}$.  We varied the learning rate over the course of training, according to the formula:
+
+\begin{equation}
+lrate = \dmodel^{-0.5} \cdot
+  \min({step\_num}^{-0.5},
+    {step\_num} \cdot {warmup\_steps}^{-1.5})
+\end{equation}
+
+This corresponds to increasing the learning rate linearly for the first $warmup\_steps$ training steps, and decreasing it thereafter proportionally to the inverse square root of the step number.  We used $warmup\_steps=4000$.
+
+\subsection{Regularization} \label{sec:reg}
+
+We employ three types of regularization during training: 
+\paragraph{Residual Dropout} We apply dropout \citep{srivastava2014dropout} to the output of each sub-layer, before it is added to the sub-layer input and normalized.   In addition, we apply dropout to the sums of the embeddings and the positional encodings in both the encoder and decoder stacks.  For the base model, we use a rate of $P_{drop}=0.1$.
+
+% \paragraph{Attention Dropout} Query to key attentions are structurally similar to hidden-to-hidden weights in a feed-forward network, albeit across positions. The softmax activations yielding attention weights can then be seen as the analogue of hidden layer activations. A natural possibility is to extend dropout \citep{srivastava2014dropout} to attention. We implement attention dropout by dropping out attention weights as,
+% \begin{equation*}
+%   \mathrm{Attention}(Q, K, V) = \mathrm{dropout}(\mathrm{softmax}(\frac{QK^T}{\sqrt{d}}))V
+% \end{equation*}
+% In addition to residual dropout, we found attention dropout to be beneficial for our parsing experiments.  
+
+%\paragraph{Symbol Dropout} In the source and target embedding layers, we replace a random subset of the token ids with a sentinel id.  For the base model, we use a rate of $symbol\_dropout\_rate=0.1$.  Note that this applies only to the auto-regressive use of the target ids - not their use in the cross-entropy loss. 
+
+%\paragraph{Attention Dropout} Query to memory attentions are structurally similar to hidden-to-hidden weights in a feed-forward network, albeit across positions. The softmax activations yielding attention weights can then be seen as the analogue of hidden layer activations. A natural possibility is to extend dropout \citep{srivastava2014dropout} to attentions. We implement attention dropout by dropping out attention weights as,
+%\begin{equation*}
+%   A(Q, K, V) = \mathrm{dropout}(\mathrm{softmax}(\frac{QK^T}{\sqrt{d}}))V
+%\end{equation*}
+%As a result, the query will not be able to access the memory values at the dropped out position. In our experiments, we tried attention dropout rates of 0.2, and 0.3, and found it to work favorably for English-to-German translation.
+%$attention\_dropout\_rate=0.2$.
+
+\paragraph{Label Smoothing} During training, we employed label smoothing of value $\epsilon_{ls}=0.1$ \citep{DBLP:journals/corr/SzegedyVISW15}.  This hurts perplexity, as the model learns to be more unsure, but improves accuracy and BLEU score.

download/20230719163556/output/Attention-Is-All-You-Need/visualizations.tex

@@ -0,0 +1,18 @@
+\pagebreak
+\section*{Attention Visualizations}\label{sec:viz-att}
+\begin{figure*}[h]
+{\includegraphics[width=\textwidth, trim=0 0 0 36, clip]{./vis/making_more_difficult5_new.pdf}}
+\caption{An example of the attention mechanism following long-distance dependencies in the encoder self-attention in layer 5 of 6. Many of the attention heads attend to a distant dependency of the verb `making', completing the phrase `making...more difficult'.  Attentions here shown only for the word `making'. Different colors represent different heads. Best viewed in color.}
+\end{figure*}
+
+\begin{figure*}
+{\includegraphics[width=\textwidth, trim=0 0 0 45, clip]{./vis/anaphora_resolution_new.pdf}}
+{\includegraphics[width=\textwidth, trim=0 0 0 37, clip]{./vis/anaphora_resolution2_new.pdf}}
+\caption{Two attention heads, also in layer 5 of 6, apparently involved in anaphora resolution. Top: Full attentions for head 5. Bottom: Isolated attentions from just the word `its' for attention heads 5 and 6. Note that the attentions are very sharp for this word.}
+\end{figure*}
+
+\begin{figure*}
+{\includegraphics[width=\textwidth, trim=0 0 0 36, clip]{./vis/attending_to_head_new.pdf}}
+{\includegraphics[width=\textwidth, trim=0 0 0 36, clip]{./vis/attending_to_head2_new.pdf}}
+\caption{Many of the attention heads exhibit behaviour that seems related to the structure of the sentence. We give two such examples above, from two different heads from the encoder self-attention at layer 5 of 6. The heads clearly learned to perform different tasks.}
+\end{figure*}

download/20230719163556/output/Attention-Is-All-You-Need/why_self_attention.tex

@@ -0,0 +1,98 @@
+%We focus on the general task of mapping one variable-length sequence of symbol representations ${x_1, ..., x_n} \in \mathbb{R}^d$ to another sequence of the same length ${y_1, ..., y_n} \in \mathbb{R}^d$. \marginpar{should we use this notation? alternatively we can just say "d-dimensional vectors"}
+
+In this section we compare various aspects of self-attention layers to the recurrent and convolutional layers commonly used for mapping one variable-length sequence of symbol representations $(x_1, ..., x_n)$ to another sequence of equal length $(z_1, ..., z_n)$, with $x_i, z_i \in \mathbb{R}^d$, such as a hidden layer in a typical sequence transduction encoder or decoder. Motivating our use of self-attention we consider three desiderata.
+
+One is the total computational complexity per layer.
+Another is the amount of computation that can be parallelized, as measured by the minimum number of sequential operations required.
+
+The third is the path length between long-range dependencies in the network. Learning long-range dependencies is a key challenge in many sequence transduction tasks. One key factor affecting the ability to learn such dependencies is the length of the paths forward and backward signals have to traverse in the network. The shorter these paths between any combination of positions in the input and output sequences, the easier it is to learn long-range dependencies \citep{hochreiter2001gradient}. Hence we also compare the maximum path length between any two input and output positions in networks composed of the different layer types.
+
+%\subsection{Computational Performance and Path Lengths}
+
+\begin{table}[t]
+\caption{
+  Maximum path lengths, per-layer complexity and minimum number of sequential operations for different layer types. $n$ is the sequence length, $d$ is the representation dimension, $k$ is the kernel size of convolutions and $r$ the size of the neighborhood in restricted self-attention.}
+  %Attention models are quite efficient for cross-positional communications when sequence length is smaller than channel depth.
+\label{tab:op_complexities}
+\begin{center}
+\vspace{-1mm}
+%\scalebox{0.75}{
+
+\begin{tabular}{lccc}
+\toprule
+Layer Type & Complexity per Layer & Sequential & Maximum Path Length  \\
+           &             & Operations &   \\
+\hline
+\rule{0pt}{2.0ex}Self-Attention & $O(n^2 \cdot d)$ & $O(1)$ & $O(1)$ \\
+Recurrent & $O(n \cdot d^2)$ & $O(n)$ & $O(n)$ \\
+
+Convolutional & $O(k \cdot n \cdot d^2)$ & $O(1)$ & $O(log_k(n))$ \\
+%\cmidrule
+Self-Attention (restricted)& $O(r \cdot n \cdot d)$ & $O(1)$ & $O(n/r)$ \\
+
+%Convolutional (separable) & $O(k \cdot n \cdot d + n \cdot d^2)$ & $O(1)$ & $O(log_k(n))$ \\
+
+%Position-wise Feed-Forward & $O(n \cdot d^2)$ & $O(1)$ & $\infty$ \\
+
+%Fully Connected & $O(n^2 \cdot d^2)$ & $O(1)$ & $O(1)$ \\
+%Convolutional (separable) & $O(k \cdot n \cdot d + n \cdot d^2)$ & $O(1)$ & $O(log_k(n))$ \\
+
+%Position-wise Feed-Forward & $O(n \cdot d^2)$ & $O(1)$ & $\infty$ \\
+
+%Fully Connected & $O(n^2 \cdot d^2)$ & $O(1)$ & $O(1)$ \\
+\bottomrule
+\end{tabular}
+%}
+\end{center}
+\end{table}
+
+
+%\begin{table}[b]
+%\caption{
+%  Maximum path lengths, per-layer complexity and minimum number of sequential operations for different layer types. $n$ is the sequence length, $d$ is the representation dimensionality, $k$ is the kernel size of convolutions and $r$ the size of the neighborhood in localized self-attention.}
+  %Attention models are quite efficient for cross-positional communications when sequence length is smaller than channel depth.
+%\label{tab:op_complexities}
+%\begin{center}
+%\vspace{-1mm}
+%%\scalebox{0.75}{
+%
+%\begin{tabular}{lccc}
+%\hline
+%Layer Type & Receptive & Complexity per Layer & Sequential  %\\
+%           & Field Size     &            & Operations  \\
+%\hline
+%Self-Attention & $n$ & $O(n^2 \cdot d)$ & $O(1)$ \\
+%Recurrent & $n$ & $O(n \cdot d^2)$ & $O(n)$ \\
+
+%Convolutional & $k$ & $O(k \cdot n \cdot d^2)$ & %$O(log_k(n))$ \\
+%\hline 
+%Self-Attention (localized)& $r$ & $O(r \cdot n \cdot d)$ & %$O(1)$ \\
+
+%Convolutional (separable) & $k$ & $O(k \cdot n \cdot d + n %\cdot d^2)$ & $O(log_k(n))$ \\
+
+%Position-wise Feed-Forward & $1$ & $O(n \cdot d^2)$ & $O(1)$ %\\
+
+%Fully Connected & $n$ & $O(n^2 \cdot d^2)$ & $O(1)$ \\
+
+%\end{tabular}
+%%}
+%\end{center}
+%\end{table}
+
+%The receptive field size of a layer is the number of different input representations that can influence any particular output representation. Recurrent layers and self-attention layers have a full receptive field equal to the sequence length $n$. Convolutional layers have a limited receptive field equal to their kernel width $k$, which is generally chosen to be small in order to limit computational cost.
+
+As noted in Table \ref{tab:op_complexities}, a self-attention layer connects all positions with a constant number of sequentially executed operations, whereas a recurrent layer requires $O(n)$ sequential operations.
+In terms of computational complexity, self-attention layers are faster than recurrent layers when the sequence length $n$ is smaller than the representation dimensionality $d$, which is most often the case with sentence representations used by state-of-the-art models in machine translations, such as word-piece \citep{wu2016google} and byte-pair \citep{sennrich2015neural} representations.
+To improve computational performance for tasks involving very long sequences, self-attention could be restricted to considering only a neighborhood of size $r$ in the input sequence centered around the respective output position. This would increase the maximum path length to $O(n/r)$. We plan to investigate this approach further in future work.
+
+A single convolutional layer with kernel width $k < n$ does not connect all pairs of input and output positions. Doing so requires a stack of $O(n/k)$ convolutional layers in the case of contiguous kernels, or $O(log_k(n))$ in the case of dilated convolutions \citep{NalBytenet2017}, increasing the length of the longest paths between any two positions in the network.
+Convolutional layers are generally more expensive than recurrent layers, by a factor of $k$. Separable convolutions \citep{xception2016}, however, decrease the complexity considerably, to $O(k \cdot n \cdot d + n \cdot d^2)$. Even with $k=n$, however, the complexity of a separable convolution is equal to the combination of a self-attention layer and a point-wise feed-forward layer, the approach we take in our model.
+
+%\subsection{Unfiltered Bottleneck Argument}
+
+%An orthogonal argument can be made for self-attention layers based on when the layer imposes the bottleneck of mapping all of the information used to compute a given output position into a single, fixed-length vector. ...
+
+%There is a second argument for self-attention layers which we call the unfiltered bottleneck argument.   In both recurrent and the convolutional layers, the information that position $i$ receives from the other positions is compressed to a vector of dimension $d$ before it ever can be filtered by the content $x_i$.  More precisely, we can express $y_i = F(i, x_i, G(i, \{x_{j \neq i}\}))$, where $G(i, \{x_{j \neq i}\})$ is a vector of dimension $d$.  Intuitively, we would expect that this would cause a large amount of irrelevant information to crowd out the relevant information.  Self-attention does not suffer from the unfiltered bottleneck problem, since the aggregation happens after filtering, and so, intuitively, we have the chance of transmitting lots of relevant information.
+
+As side benefit, self-attention could yield more interpretable models. We inspect attention distributions from our models and present and discuss examples in the appendix. Not only do individual attention heads clearly learn to perform different tasks, many appear to exhibit behavior related to the syntactic and semantic structure of the sentences.
+

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/cvpr.sty

@@ -0,0 +1,249 @@
+% ---------------------------------------------------------------
+%
+% $Id: cvpr.sty,v 1.3 2005/10/24 19:56:15 awf Exp $
+%
+% by [email protected]
+% some mods by [email protected]
+%
+% ---------------------------------------------------------------
+%
+% no guarantee is given that the format corresponds perfectly to
+% IEEE 8.5" x 11" Proceedings, but most features should be ok.
+%
+% ---------------------------------------------------------------
+% with LaTeX2e:
+% =============
+%
+% use as
+%   \documentclass[times,10pt,twocolumn]{article}
+%   \usepackage{latex8}
+%   \usepackage{times}
+%
+% ---------------------------------------------------------------
+
+% with LaTeX 2.09:
+% ================
+%
+% use as
+%   \documentstyle[times,art10,twocolumn,latex8]{article}
+%
+% ---------------------------------------------------------------
+% with both versions:
+% ===================
+%
+% specify \cvprfinalcopy to emit the final camera-ready copy
+%
+% specify references as
+%   \bibliographystyle{ieee}
+%   \bibliography{...your files...}
+%
+% ---------------------------------------------------------------
+
+\usepackage{eso-pic}
+\usepackage{xspace}
+
+\typeout{CVPR 8.5 x 11-Inch Proceedings Style `cvpr.sty'.}
+
+% ten point helvetica bold required for captions
+% eleven point times bold required for second-order headings
+% in some sites the name of the fonts may differ,
+% change the name here:
+\font\cvprtenhv  = phvb at 8pt % *** IF THIS FAILS, SEE cvpr.sty ***
+\font\elvbf  = ptmb scaled 1100
+
+% If the above lines give an error message, try to comment them and
+% uncomment these:
+%\font\cvprtenhv  = phvb7t at 8pt
+%\font\elvbf  = ptmb7t scaled 1100
+
+% set dimensions of columns, gap between columns, and paragraph indent
+\setlength{\textheight}{8.875in}
+\setlength{\textwidth}{6.875in}
+\setlength{\columnsep}{0.3125in}
+\setlength{\topmargin}{0in}
+\setlength{\headheight}{0in}
+\setlength{\headsep}{0in}
+\setlength{\parindent}{1pc}
+\setlength{\oddsidemargin}{-.304in}
+\setlength{\evensidemargin}{-.304in}
+
+\newif\ifcvprfinal
+\cvprfinalfalse
+\def\cvprfinalcopy{\global\cvprfinaltrue}
+
+% memento from size10.clo
+% \normalsize{\@setfontsize\normalsize\@xpt\@xiipt}
+% \small{\@setfontsize\small\@ixpt{11}}
+% \footnotesize{\@setfontsize\footnotesize\@viiipt{9.5}}
+% \scriptsize{\@setfontsize\scriptsize\@viipt\@viiipt}
+% \tiny{\@setfontsize\tiny\@vpt\@vipt}
+% \large{\@setfontsize\large\@xiipt{14}}
+% \Large{\@setfontsize\Large\@xivpt{18}}
+% \LARGE{\@setfontsize\LARGE\@xviipt{22}}
+% \huge{\@setfontsize\huge\@xxpt{25}}
+% \Huge{\@setfontsize\Huge\@xxvpt{30}}
+
+\def\@maketitle
+   {
+   \newpage
+   \null
+   \vskip .375in
+   \begin{center}
+      {\Large \bf \@title \par}
+      % additional two empty lines at the end of the title
+      \vspace*{24pt}
+      {
+      \large
+      \lineskip .5em
+      \begin{tabular}[t]{c}
+         \ifcvprfinal\@author\else Anonymous CVPR submission\\
+         \vspace*{1pt}\\%This space will need to be here in the final copy, so don't squeeze it out for the review copy.
+Paper ID \cvprPaperID \fi
+      \end{tabular}
+      \par
+      }
+      % additional small space at the end of the author name
+      \vskip .5em
+      % additional empty line at the end of the title block
+      \vspace*{12pt}
+   \end{center}
+   }
+
+\def\abstract
+   {%
+   \centerline{\large\bf Abstract}%
+   \vspace*{12pt}%
+   \it%
+   }
+
+\def\endabstract
+   {
+   % additional empty line at the end of the abstract
+   \vspace*{12pt}
+   }
+
+\def\affiliation#1{\gdef\@affiliation{#1}} \gdef\@affiliation{}
+
+\newlength{\@ctmp}
+\newlength{\@figindent}
+\setlength{\@figindent}{1pc}
+
+\long\def\@makecaption#1#2{
+   \setbox\@tempboxa\hbox{\small \noindent #1.~#2}
+   \setlength{\@ctmp}{\hsize}
+   \addtolength{\@ctmp}{-\@figindent}\addtolength{\@ctmp}{-\@figindent}
+   % IF longer than one indented paragraph line
+   \ifdim \wd\@tempboxa >\@ctmp
+      % THEN DON'T set as an indented paragraph
+      {\small #1.~#2\par}
+   \else
+      % ELSE center
+      \hbox to\hsize{\hfil\box\@tempboxa\hfil}
+  \fi}
+
+% correct heading spacing and type
+\def\cvprsection{\@startsection {section}{1}{\z@}
+   {10pt plus 2pt minus 2pt}{7pt} {\large\bf}}
+\def\cvprssect#1{\cvprsection*{#1}}
+\def\cvprsect#1{\cvprsection{\hskip -1em.~#1}}
+\def\section{\@ifstar\cvprssect\cvprsect}
+
+\def\cvprsubsection{\@startsection {subsection}{2}{\z@}
+   {8pt plus 2pt minus 2pt}{6pt} {\elvbf}}
+\def\cvprssubsect#1{\cvprsubsection*{#1}}
+\def\cvprsubsect#1{\cvprsubsection{\hskip -1em.~#1}}
+\def\subsection{\@ifstar\cvprssubsect\cvprsubsect}
+
+%% --------- Page background marks: Ruler and confidentiality
+
+% ----- define vruler
+\makeatletter
+\newbox\cvprrulerbox
+\newcount\cvprrulercount
+\newdimen\cvprruleroffset
+\newdimen\cv@lineheight
+\newdimen\cv@boxheight
+\newbox\cv@tmpbox
+\newcount\cv@refno
+\newcount\cv@tot
+% NUMBER with left flushed zeros  \fillzeros[<WIDTH>]<NUMBER>
+\newcount\cv@tmpc@ \newcount\cv@tmpc
+\def\fillzeros[#1]#2{\cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi
+\cv@tmpc=1 %
+\loop\ifnum\cv@tmpc@<10 \else \divide\cv@tmpc@ by 10 \advance\cv@tmpc by 1 \fi
+   \ifnum\cv@tmpc@=10\relax\cv@tmpc@=11\relax\fi \ifnum\cv@tmpc@>10 \repeat
+\ifnum#2<0\advance\cv@tmpc1\relax-\fi
+\loop\ifnum\cv@tmpc<#1\relax0\advance\cv@tmpc1\relax\fi \ifnum\cv@tmpc<#1 \repeat
+\cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi \relax\the\cv@tmpc@}%
+% \makevruler[<SCALE>][<INITIAL_COUNT>][<STEP>][<DIGITS>][<HEIGHT>]
+\def\makevruler[#1][#2][#3][#4][#5]{\begingroup\offinterlineskip
+\textheight=#5\vbadness=10000\vfuzz=120ex\overfullrule=0pt%
+\global\setbox\cvprrulerbox=\vbox to \textheight{%
+{\parskip=0pt\hfuzz=150em\cv@boxheight=\textheight
+\cv@lineheight=#1\global\cvprrulercount=#2%
+\cv@tot\cv@boxheight\divide\cv@tot\cv@lineheight\advance\cv@tot2%
+\cv@refno1\vskip-\cv@lineheight\vskip1ex%
+\loop\setbox\cv@tmpbox=\hbox to0cm{{\cvprtenhv\hfil\fillzeros[#4]\cvprrulercount}}%
+\ht\cv@tmpbox\cv@lineheight\dp\cv@tmpbox0pt\box\cv@tmpbox\break
+\advance\cv@refno1\global\advance\cvprrulercount#3\relax
+\ifnum\cv@refno<\cv@tot\repeat}}\endgroup}%
+\makeatother
+% ----- end of vruler
+
+% \makevruler[<SCALE>][<INITIAL_COUNT>][<STEP>][<DIGITS>][<HEIGHT>]
+\def\cvprruler#1{\makevruler[12pt][#1][1][3][0.993\textheight]\usebox{\cvprrulerbox}}
+\AddToShipoutPicture{%
+\ifcvprfinal\else
+  %\AtTextLowerLeft{%
+  % \color[gray]{.15}\framebox(\LenToUnit{\textwidth},\LenToUnit{\textheight}){}
+  %}
+\cvprruleroffset=\textheight
+\advance\cvprruleroffset by -3.7pt
+  \color[rgb]{.5,.5,1}
+  \AtTextUpperLeft{%
+    \put(\LenToUnit{-35pt},\LenToUnit{-\cvprruleroffset}){%left ruler
+      \cvprruler{\cvprrulercount}}
+    \put(\LenToUnit{\textwidth\kern 30pt},\LenToUnit{-\cvprruleroffset}){%right ruler
+      \cvprruler{\cvprrulercount}}
+  }
+\def\pid{\parbox{1in}{\begin{center}\bf\sf{\small CVPR}\\\#\cvprPaperID\end{center}}}
+  \AtTextUpperLeft{%paperID in corners
+    \put(\LenToUnit{-65pt},\LenToUnit{45pt}){\pid}
+    \put(\LenToUnit{\textwidth\kern-8pt},\LenToUnit{45pt}){\pid}
+  }
+  \AtTextUpperLeft{%confidential
+    \put(0,\LenToUnit{1cm}){\parbox{\textwidth}{\centering\cvprtenhv
+       CVPR 2016 Submission \#\cvprPaperID. CONFIDENTIAL REVIEW COPY.  DO NOT DISTRIBUTE.}}
+  }
+\fi
+}
+
+%%% Make figure placement a little more predictable.
+% We trust the user to move figures if this results
+% in ugliness.
+% Minimize bad page breaks at figures
+\renewcommand{\textfraction}{0.01}
+\renewcommand{\floatpagefraction}{0.99}
+\renewcommand{\topfraction}{0.99}
+\renewcommand{\bottomfraction}{0.99}
+\renewcommand{\dblfloatpagefraction}{0.99}
+\renewcommand{\dbltopfraction}{0.99}
+\setcounter{totalnumber}{99}
+\setcounter{topnumber}{99}
+\setcounter{bottomnumber}{99}
+
+% Add a period to the end of an abbreviation unless there's one
+% already, then \xspace.
+\makeatletter
+\DeclareRobustCommand\onedot{\futurelet\@let@token\@onedot}
+\def\@onedot{\ifx\@let@token.\else.\null\fi\xspace}
+
+\def\eg{\emph{e.g}\onedot} \def\Eg{\emph{E.g}\onedot}
+\def\ie{\emph{i.e}\onedot} \def\Ie{\emph{I.e}\onedot}
+\def\cf{\emph{c.f}\onedot} \def\Cf{\emph{C.f}\onedot}
+\def\etc{\emph{etc}\onedot} \def\vs{\emph{vs}\onedot}
+\def\wrt{w.r.t\onedot} \def\dof{d.o.f\onedot}
+\def\etal{\emph{et al}\onedot}
+\makeatother
+
+% ---------------------------------------------------------------

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/cvpr_eso.sty

@@ -0,0 +1,109 @@
+%%
+%% This is file `everyshi.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% everyshi.dtx  (with options: `package')
+%% 
+%% Copyright (C) [1994..1999] by Martin Schroeder. All rights reserved.
+%% 
+%% This file is part of the EveryShi package
+%% 
+%% This program may be redistributed and/or modified under the terms
+%% of the LaTeX Project Public License, either version 1.0 of this
+%% license, or (at your option) any later version.
+%% The latest version of this license is in
+%%    CTAN:macros/latex/base/lppl.txt.
+%% 
+%% Happy users are requested to send me a postcard. :-)
+%% 
+%% The EveryShi package contains these files:
+%% 
+%%       everyshi.asc
+%%       everyshi.dtx
+%%       everyshi.dvi
+%%       everyshi.ins
+%%       everyshi.bug
+%% 
+%% Error Reports in case of UNCHANGED versions to
+%% 
+%%                         Martin Schr"oder
+%%                         Cr"usemannallee 3
+%%                         D-28213 Bremen
+%%                         [email protected]
+%% 
+%% File: everyshi.dtx Copyright (C) 2001 Martin Schr\"oder
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesPackage{everyshi}
+         [2001/05/15 v3.00 EveryShipout Package (MS)]
+%% \CharacterTable
+%% {Upper-case    \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z
+%%  Lower-case    \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z
+%%  Digits        \0\1\2\3\4\5\6\7\8\9
+%%  Exclamation   \!     Double quote  \"     Hash (number) \#
+%%  Dollar        \$     Percent       \%     Ampersand     \&
+%%  Acute accent  \'     Left paren    \(     Right paren   \)
+%%  Asterisk      \*     Plus          \+     Comma         \,
+%%  Minus         \-     Point         \.     Solidus       \/
+%%  Colon         \:     Semicolon     \;     Less than     \<
+%%  Equals        \=     Greater than  \>     Question mark \?
+%%  Commercial at \@     Left bracket  \[     Backslash     \\
+%%  Right bracket \]     Circumflex    \^     Underscore    \_
+%%  Grave accent  \`     Left brace    \{     Vertical bar  \|
+%%  Right brace   \}     Tilde         \~}
+%%
+%% \iffalse meta-comment
+%% ===================================================================
+%%  @LaTeX-package-file{
+%%     author          = {Martin Schr\"oder},
+%%     version         = "3.00",
+%%     date            = "15 May 2001",
+%%     filename        = "everyshi.sty",
+%%     address         = {Martin Schr\"oder
+%%                        Cr\"usemannallee 3
+%%                        28213 Bremen
+%%                        Germany},
+%%     telephone       = "+49-421-2239425",
+%%     email           = "[email protected]",
+%%     pgp-Key         = "2048 bit / KeyID 292814E5",
+%%     pgp-fingerprint = "7E86 6EC8 97FA 2995 82C3  FEA5 2719 090E",
+%%     docstring       = "LaTeX package which provides hooks into
+%%                        \cs{shipout}.
+%%  }
+%% ===================================================================
+%% \fi
+
+\newcommand{\@EveryShipout@Hook}{}
+\newcommand{\@EveryShipout@AtNextHook}{}
+\newcommand*{\EveryShipout}[1]
+   {\g@addto@macro\@EveryShipout@Hook{#1}}
+\newcommand*{\AtNextShipout}[1]
+   {\g@addto@macro\@EveryShipout@AtNextHook{#1}}
+\newcommand{\@EveryShipout@Shipout}{%
+   \afterassignment\@EveryShipout@Test
+   \global\setbox\@cclv= %
+   }
+\newcommand{\@EveryShipout@Test}{%
+   \ifvoid\@cclv\relax
+      \aftergroup\@EveryShipout@Output
+   \else
+      \@EveryShipout@Output
+   \fi%
+   }
+\newcommand{\@EveryShipout@Output}{%
+   \@EveryShipout@Hook%
+   \@EveryShipout@AtNextHook%
+      \gdef\@EveryShipout@AtNextHook{}%
+   \@EveryShipout@Org@Shipout\box\@cclv%
+   }
+\newcommand{\@EveryShipout@Org@Shipout}{}
+\newcommand*{\@EveryShipout@Init}{%
+   \message{ABD: EveryShipout initializing macros}%
+   \let\@EveryShipout@Org@Shipout\shipout
+   \let\shipout\@EveryShipout@Shipout
+   }
+\AtBeginDocument{\@EveryShipout@Init}
+\endinput
+%%
+%% End of file `everyshi.sty'.

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/eso-pic.sty

@@ -0,0 +1,267 @@
+%%
+%% This is file `eso-pic.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% eso-pic.dtx  (with options: `package')
+%% 
+%% This is a generated file.
+%% 
+%% Copyright (C) 1998-2002 by Rolf Niepraschk <[email protected]>
+%% 
+%% This file may be distributed and/or modified under the conditions of
+%% the LaTeX Project Public License, either version 1.2 of this license
+%% or (at your option) any later version.  The latest version of this
+%% license is in:
+%% 
+%%    http://www.latex-project.org/lppl.txt
+%% 
+%% and version 1.2 or later is part of all distributions of LaTeX version
+%% 1999/12/01 or later.
+%% 
+\NeedsTeXFormat{LaTeX2e}[1999/12/01]
+\ProvidesPackage{eso-pic}
+   [2002/11/16 v1.1b eso-pic (RN)]
+\input{cvpr_eso.sty}
+\newcommand\LenToUnit[1]{#1\@gobble}
+
+\newcommand\AtPageUpperLeft[1]{%
+  \begingroup
+    \@tempdima=0pt\relax\@tempdimb=\ESO@yoffsetI\relax
+    \put(\LenToUnit{\@tempdima},\LenToUnit{\@tempdimb}){#1}%
+  \endgroup
+}
+\newcommand\AtPageLowerLeft[1]{\AtPageUpperLeft{%
+  \put(0,\LenToUnit{-\paperheight}){#1}}}
+\newcommand\AtPageCenter[1]{\AtPageUpperLeft{%
+  \put(\LenToUnit{.5\paperwidth},\LenToUnit{-.5\paperheight}){#1}}%
+}
+\newcommand\AtTextUpperLeft[1]{%
+  \begingroup
+    \setlength\@tempdima{1in}%
+    \ifodd\c@page%
+      \advance\@tempdima\oddsidemargin%
+    \else%
+      \advance\@tempdima\evensidemargin%
+    \fi%
+    \@tempdimb=\ESO@yoffsetI\relax\advance\@tempdimb-1in\relax%
+    \advance\@tempdimb-\topmargin%
+    \advance\@tempdimb-\headheight\advance\@tempdimb-\headsep%
+    \put(\LenToUnit{\@tempdima},\LenToUnit{\@tempdimb}){#1}%
+  \endgroup
+}
+\newcommand\AtTextLowerLeft[1]{\AtTextUpperLeft{%
+  \put(0,\LenToUnit{-\textheight}){#1}}}
+\newcommand\AtTextCenter[1]{\AtTextUpperLeft{%
+  \put(\LenToUnit{.5\textwidth},\LenToUnit{-.5\textheight}){#1}}}
+\newcommand{\ESO@HookI}{} \newcommand{\ESO@HookII}{}
+\newcommand{\ESO@HookIII}{}
+\newcommand{\AddToShipoutPicture}{%
+  \@ifstar{\g@addto@macro\ESO@HookII}{\g@addto@macro\ESO@HookI}}
+\newcommand{\ClearShipoutPicture}{\global\let\ESO@HookI\@empty}
+\newcommand\ESO@isMEMOIR[1]{}
+\@ifclassloaded{memoir}{\renewcommand\ESO@isMEMOIR[1]{#1}}{}
+\newcommand{\@ShipoutPicture}{%
+  \bgroup
+    \@tempswafalse%
+    \ifx\ESO@HookI\@empty\else\@tempswatrue\fi%
+    \ifx\ESO@HookII\@empty\else\@tempswatrue\fi%
+    \ifx\ESO@HookIII\@empty\else\@tempswatrue\fi%
+    \if@tempswa%
+      \@tempdima=1in\@tempdimb=-\@tempdima%
+      \advance\@tempdimb\ESO@yoffsetI%
+      \ESO@isMEMOIR{%
+        \advance\@tempdima\trimedge%
+        \advance\@tempdima\paperwidth%
+        \advance\@tempdima-\stockwidth%
+        \if@twoside\ifodd\c@page\else%
+          \advance\@tempdima-2\trimedge%
+          \advance\@tempdima-\paperwidth%
+          \advance\@tempdima\stockwidth%
+        \fi\fi%
+        \advance\@tempdimb\trimtop}%
+      \unitlength=1pt%
+      \global\setbox\@cclv\vbox{%
+        \vbox{\let\protect\relax
+          \pictur@(0,0)(\strip@pt\@tempdima,\strip@pt\@tempdimb)%
+            \ESO@HookIII\ESO@HookI\ESO@HookII%
+            \global\let\ESO@HookII\@empty%
+          \endpicture}%
+          \nointerlineskip%
+        \box\@cclv}%
+    \fi
+  \egroup
+}
+\EveryShipout{\@ShipoutPicture}
+\RequirePackage{keyval}
+\newif\ifESO@dvips\ESO@dvipsfalse \newif\ifESO@grid\ESO@gridfalse
+\newif\ifESO@texcoord\ESO@texcoordfalse
+\newcommand*\ESO@gridunitname{}
+\newcommand*\ESO@gridunit{}
+\newcommand*\ESO@labelfactor{}
+\newcommand*\ESO@griddelta{}\newcommand*\ESO@griddeltaY{}
+\newcommand*\ESO@gridDelta{}\newcommand*\ESO@gridDeltaY{}
+\newcommand*\ESO@gridcolor{}
+\newcommand*\ESO@subgridcolor{}
+\newcommand*\ESO@subgridstyle{dotted}% ???
+\newcommand*\ESO@gap{}
+\newcommand*\ESO@yoffsetI{}\newcommand*\ESO@yoffsetII{}
+\newcommand*\ESO@gridlines{\thinlines}
+\newcommand*\ESO@subgridlines{\thinlines}
+\newcommand*\ESO@hline[1]{\ESO@subgridlines\line(1,0){#1}}
+\newcommand*\ESO@vline[1]{\ESO@subgridlines\line(0,1){#1}}
+\newcommand*\ESO@Hline[1]{\ESO@gridlines\line(1,0){#1}}
+\newcommand*\ESO@Vline[1]{\ESO@gridlines\line(0,1){#1}}
+\newcommand\ESO@fcolorbox[4][]{\fbox{#4}}
+\newcommand\ESO@color[1]{}
+\newcommand\ESO@colorbox[3][]{%
+  \begingroup
+    \fboxrule=0pt\fbox{#3}%
+  \endgroup
+}
+\newcommand\gridSetup[6][]{%
+  \edef\ESO@gridunitname{#1}\edef\ESO@gridunit{#2}
+  \edef\ESO@labelfactor{#3}\edef\ESO@griddelta{#4}
+  \edef\ESO@gridDelta{#5}\edef\ESO@gap{#6}}
+\define@key{ESO}{texcoord}[true]{\csname ESO@texcoord#1\endcsname}
+\define@key{ESO}{pscoord}[true]{\csname @tempswa#1\endcsname
+  \if@tempswa\ESO@texcoordfalse\else\ESO@texcoordtrue\fi}
+\define@key{ESO}{dvips}[true]{\csname ESO@dvips#1\endcsname}
+\define@key{ESO}{grid}[true]{\csname ESO@grid#1\endcsname
+  \setkeys{ESO}{gridcolor=black,subgridcolor=black}}
+\define@key{ESO}{colorgrid}[true]{\csname ESO@grid#1\endcsname
+  \setkeys{ESO}{gridcolor=red,subgridcolor=green}}
+\define@key{ESO}{gridcolor}{\def\ESO@gridcolor{#1}}
+\define@key{ESO}{subgridcolor}{\def\ESO@subgridcolor{#1}}
+\define@key{ESO}{subgridstyle}{\def\ESO@subgridstyle{#1}}%
+\define@key{ESO}{gridunit}{%
+  \def\@tempa{#1}
+  \def\@tempb{bp}
+  \ifx\@tempa\@tempb
+    \gridSetup[\@tempa]{1bp}{1}{10}{50}{2}
+  \else
+    \def\@tempb{pt}
+    \ifx\@tempa\@tempb
+      \gridSetup[\@tempa]{1pt}{1}{10}{50}{2}
+    \else
+      \def\@tempb{in}
+      \ifx\@tempa\@tempb
+        \gridSetup[\@tempa]{.1in}{.1}{2}{10}{.5}
+      \else
+        \gridSetup[mm]{1mm}{1}{5}{20}{1}
+      \fi
+    \fi
+  \fi
+}
+\setkeys{ESO}{subgridstyle=solid,pscoord=true,gridunit=mm}
+\def\ProcessOptionsWithKV#1{%
+  \let\@tempc\@empty
+  \@for\CurrentOption:=\@classoptionslist\do{%
+    \@ifundefined{KV@#1@\CurrentOption}%
+    {}{\edef\@tempc{\@tempc,\CurrentOption,}}}%
+  \edef\@tempc{%
+    \noexpand\setkeys{#1}{\@tempc\@ptionlist{\@currname.\@currext}}}%
+  \@tempc
+  \AtEndOfPackage{\let\@unprocessedoptions\relax}}%
+\ProcessOptionsWithKV{ESO}%
+\newcommand\ESO@div[2]{%
+  \@tempdima=#1\relax\@tempdimb=\ESO@gridunit\relax
+  \@tempdimb=#2\@tempdimb\divide\@tempdima by \@tempdimb%
+  \@tempcnta\@tempdima\advance\@tempcnta\@ne}
+\AtBeginDocument{%
+  \IfFileExists{color.sty}
+  {%
+    \RequirePackage{color}
+    \let\ESO@color=\color\let\ESO@colorbox=\colorbox
+    \let\ESO@fcolorbox=\fcolorbox
+  }{}
+  \@ifundefined{Gin@driver}{}%
+  {%
+    \ifx\Gin@driver\@empty\else%
+      \filename@parse{\Gin@driver}\def\reserved@a{dvips}%
+      \ifx\filename@base\reserved@a\ESO@dvipstrue\fi%
+    \fi
+  }%
+  \ifx\pdfoutput\undefined\else
+    \ifx\pdfoutput\relax\else
+      \ifcase\pdfoutput\else
+        \ESO@dvipsfalse%
+      \fi
+    \fi
+  \fi
+  \ifESO@dvips\def\@tempb{eepic}\else\def\@tempb{epic}\fi
+  \def\@tempa{dotted}%\def\ESO@gap{\LenToUnit{6\@wholewidth}}%
+  \ifx\@tempa\ESO@subgridstyle
+    \IfFileExists{\@tempb.sty}%
+    {%
+      \RequirePackage{\@tempb}
+      \renewcommand*\ESO@hline[1]{\ESO@subgridlines\dottedline{\ESO@gap}%
+        (0,0)(##1,0)}
+      \renewcommand*\ESO@vline[1]{\ESO@subgridlines\dottedline{\ESO@gap}%
+        (0,0)(0,##1)}
+    }{}
+  \else
+    \ifx\ESO@gridcolor\ESO@subgridcolor%
+      \renewcommand*\ESO@gridlines{\thicklines}
+    \fi
+  \fi
+}
+\ifESO@texcoord
+  \def\ESO@yoffsetI{0pt}\def\ESO@yoffsetII{-\paperheight}
+  \edef\ESO@griddeltaY{-\ESO@griddelta}\edef\ESO@gridDeltaY{-\ESO@gridDelta}
+\else
+  \def\ESO@yoffsetI{\paperheight}\def\ESO@yoffsetII{0pt}
+  \edef\ESO@griddeltaY{\ESO@griddelta}\edef\ESO@gridDeltaY{\ESO@gridDelta}
+\fi
+\newcommand\ESO@gridpicture{%
+  \begingroup
+    \setlength\unitlength{\ESO@gridunit}%
+    \ESO@color{\ESO@subgridcolor}%
+    \ESO@div{\paperheight}{\ESO@griddelta}%
+    \multiput(0,0)(0,\ESO@griddeltaY){\@tempcnta}%
+      {\ESO@hline{\LenToUnit{\paperwidth}}}%
+    \ESO@div{\paperwidth}{\ESO@griddelta}%
+    \multiput(0,\LenToUnit{\ESO@yoffsetII})(\ESO@griddelta,0){\@tempcnta}%
+      {\ESO@vline{\LenToUnit{\paperheight}}}%
+    \ESO@color{\ESO@gridcolor}%
+    \ESO@div{\paperheight}{\ESO@gridDelta}%
+    \multiput(0,0)(0,\ESO@gridDeltaY){\@tempcnta}%
+      {\ESO@Hline{\LenToUnit{\paperwidth}}}%
+    \ESO@div{\paperwidth}{\ESO@gridDelta}%
+    \multiput(0,\LenToUnit{\ESO@yoffsetII})(\ESO@gridDelta,0){\@tempcnta}%
+      {\ESO@Vline{\LenToUnit{\paperheight}}}%
+    \fontsize{10}{12}\normalfont%
+    \ESO@div{\paperwidth}{\ESO@gridDelta}%
+    \multiput(0,\ESO@gridDeltaY)(\ESO@gridDelta,0){\@tempcnta}{%
+      \@tempcntb=\@tempcnta\advance\@tempcntb-\@multicnt%
+      \ifnum\@tempcntb>1\relax
+        \multiply\@tempcntb by \ESO@gridDelta\relax%
+        \@tempdima=\@tempcntb sp\@tempdima=\ESO@labelfactor\@tempdima%
+        \@tempcntb=\@tempdima%
+        \makebox(0,0)[c]{\ESO@colorbox{white}{\the\@tempcntb}}%
+      \fi}%
+    \ifx\ESO@gridunitname\@empty\def\@tempa{0}\else\def\@tempa{1}\fi%
+    \ESO@div{\paperheight}{\ESO@gridDelta}%
+    \multiput(\ESO@gridDelta,0)(0,\ESO@gridDeltaY){\@tempcnta}{%
+      \@tempcntb=\@tempcnta\advance\@tempcntb-\@multicnt%
+      \ifnum\@tempcntb>\@tempa\relax
+        \multiply\@tempcntb by \ESO@gridDelta\relax%
+        \@tempdima=\@tempcntb sp\@tempdima=\ESO@labelfactor\@tempdima%
+        \@tempcntb=\@tempdima%
+        \makebox(0,0)[c]{\ESO@colorbox{white}{\the\@tempcntb}}%
+      \fi
+    }%
+    \ifx\ESO@gridunitname\@empty\else%
+      \thicklines\fboxrule=\@wholewidth%
+      \put(\ESO@gridDelta,\ESO@gridDeltaY){\makebox(0,0)[c]{%
+        \ESO@fcolorbox{\ESO@gridcolor}{white}{%
+          \textbf{\ESO@gridunitname}}}}%
+    \fi
+    \normalcolor%
+  \endgroup
+}
+\ifESO@grid\g@addto@macro\ESO@HookIII{\ESO@gridpicture}\fi
+\endinput
+%%
+%% End of file `eso-pic.sty'.

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/ieee.bst

@@ -0,0 +1,1129 @@
+
+% ---------------------------------------------------------------
+%
+% ieee.bst,v 1.0 2002/04/16
+%
+% by Glenn Paulley ([email protected])
+%
+% Modified from latex8.bst 1995/09/15 15:13:49 ienne Exp $
+%
+% by [email protected]
+%
+%
+% ---------------------------------------------------------------
+%
+% no guarantee is given that the format corresponds perfectly to 
+% IEEE 8.5" x 11" Proceedings, but most features should be ok.
+%
+% ---------------------------------------------------------------
+%
+% `ieee' from BibTeX standard bibliography style `abbrv'
+% version 0.99a for BibTeX versions 0.99a or later, LaTeX version 2.09.
+% Copyright (C) 1985, all rights reserved.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+% The file btxbst.doc has the documentation for this style.
+
+ENTRY
+  { address
+    author
+    booktitle
+    chapter
+    edition
+    editor
+    howpublished
+    institution
+    journal
+    key
+    month
+    note
+    number
+    organization
+    pages
+    publisher
+    school
+    series
+    title
+    type
+    volume
+    year
+  }
+  {}
+  { label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+  #1 'mid.sentence :=
+  #2 'after.sentence :=
+  #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+  output.state mid.sentence =
+    { ", " * write$ }
+    { output.state after.block =
+ { add.period$ write$
+   newline$
+   "\newblock " write$
+ }
+ { output.state before.all =
+     'write$
+     { add.period$ " " * write$ }
+   if$
+ }
+      if$
+      mid.sentence 'output.state :=
+    }
+  if$
+  s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+    'pop$
+    'output.nonnull
+  if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+  duplicate$ empty$
+    { pop$ "empty " t * " in " * cite$ * warning$ }
+    'output.nonnull
+  if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+  "\bibitem{" write$
+  cite$ write$
+  "}" write$
+  newline$
+  ""
+  before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+  write$
+  newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+    'skip$
+    { after.block 'output.state := }
+  if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+    'skip$
+    { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+      if$
+    }
+  if$
+}
+
+FUNCTION {not}
+{   { #0 }
+    { #1 }
+  if$
+}
+
+FUNCTION {and}
+{   'skip$
+    { pop$ #0 }
+  if$
+}
+
+FUNCTION {or}
+{   { pop$ #1 }
+    'skip$
+  if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+    'skip$
+    'new.block
+  if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+  swap$ empty$
+  and
+    'skip$
+    'new.block
+  if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+    'skip$
+    'new.sentence
+  if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+  swap$ empty$
+  and
+    'skip$
+    'new.sentence
+  if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+    { pop$ "" }
+    'skip$
+  if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+    { pop$ "" }
+    { "{\em " swap$ * "}" * }
+  if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+  #1 'nameptr :=
+  s num.names$ 'numnames :=
+  numnames 'namesleft :=
+    { namesleft #0 > }
+    { s nameptr "{f.~}{vv~}{ll}{, jj}" format.name$ 't :=
+      nameptr #1 >
+ { namesleft #1 >
+     { ", " * t * }
+     { numnames #2 >
+  { "," * }
+  'skip$
+       if$
+       t "others" =
+  { " et~al." * }
+  { " and " * t * }
+       if$
+     }
+   if$
+ }
+ 't
+      if$
+      nameptr #1 + 'nameptr :=
+
+      namesleft #1 - 'namesleft :=
+    }
+  while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+    { "" }
+    { author format.names }
+  if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+    { "" }
+    { editor format.names
+      editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+    { "" }
+    { title "t" change.case$ }
+  if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+  ""
+    { t empty$ not }
+    { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+     { "--" *
+       t #2 global.max$ substring$ 't :=
+     }
+     {   { t #1 #1 substring$ "-" = }
+  { "-" *
+    t #2 global.max$ substring$ 't :=
+  }
+       while$
+     }
+   if$
+ }
+ { t #1 #1 substring$ *
+   t #2 global.max$ substring$ 't :=
+ }
+      if$
+    }
+  while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+    { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+   month
+ }
+      if$
+    }
+    { month empty$
+ 'year
+ { month " " * year * }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+    { "~" }
+    { " " }
+  if$
+  swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+    'pop$
+    { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+  if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+    { "" }
+    { "volume" volume tie.or.space.connect
+      series empty$
+ 'skip$
+ { " of " * series emphasize * }
+      if$
+      "volume and number" number either.or.check
+    }
+  if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+    { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+     { "number" }
+     { "Number" }
+   if$
+   number tie.or.space.connect
+   series empty$
+     { "there's a number but no series in " cite$ * warning$ }
+     { " in " * series * }
+   if$
+ }
+      if$
+    }
+    { "" }
+  if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+    { "" }
+    { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+      if$
+    }
+  if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+  #0 'multiresult :=
+    { multiresult not
+      t empty$ not
+      and
+    }
+    { t #1 #1 substring$
+      duplicate$ "-" =
+      swap$ duplicate$ "," =
+      swap$ "+" =
+      or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+      if$
+    }
+  while$
+  multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+    { "" }
+    { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+  number empty$
+    'skip$
+    { "(" number * ")" * *
+      volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+      if$
+    }
+  if$
+  pages empty$
+    'skip$
+    { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+    'format.pages
+    { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+      if$
+      chapter tie.or.space.connect
+      pages empty$
+ 'skip$
+ { ", " * format.pages * }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+    { "" }
+    { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+      if$
+    }
+  if$
+}
+
+FUNCTION {empty.misc.check}
+
+{ author empty$ title empty$ howpublished empty$
+  month empty$ year empty$ note empty$
+  and and and and and
+  key empty$ not and
+    { "all relevant fields are empty in " cite$ * warning$ }
+    'skip$
+  if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+    'skip$
+    { pop$
+      type "t" change.case$
+    }
+  if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+    { "Technical Report" }
+    'type
+  if$
+  number empty$
+    { "t" change.case$ }
+    { number tie.or.space.connect }
+  if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+    { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+   warning$
+   ""
+ }
+ { "In {\em " journal * "\/}" * }
+      if$
+    }
+    { "In " key * }
+  if$
+  " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+  editor num.names$ duplicate$
+  #2 >
+    { pop$ " et~al." * }
+    { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+     { " et~al." * }
+     { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+   if$
+ }
+      if$
+    }
+  if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+    { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+      "In "
+    }
+    { "Volume" volume tie.or.space.connect
+      " of " *
+    }
+  if$
+  editor empty$
+  editor field.or.null author field.or.null =
+  or
+    { key empty$
+ { series empty$
+     { "need editor, key, or series for " cite$ * " to crossref " *
+       crossref * warning$
+       "" *
+     }
+     { "{\em " * series * "\/}" * }
+   if$
+ }
+ { key * }
+      if$
+    }
+    { format.crossref.editor * }
+  if$
+  " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+  editor field.or.null author field.or.null =
+  or
+    { key empty$
+ { booktitle empty$
+     { "need editor, key, or booktitle for " cite$ * " to crossref " *
+       crossref * warning$
+       ""
+     }
+     { "In {\em " booktitle * "\/}" * }
+   if$
+ }
+ { "In " key * }
+      if$
+    }
+    { "In " format.crossref.editor * }
+  if$
+  " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  crossref missing$
+    { journal emphasize "journal" output.check
+      format.vol.num.pages output
+      format.date "year" output.check
+    }
+    { format.article.crossref output.nonnull
+      format.pages output
+    }
+  if$
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+  author empty$
+    { format.editors "author and editor" output.check }
+    { format.authors output.nonnull
+      crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+      if$
+    }
+  if$
+  new.block
+  format.btitle "title" output.check
+  crossref missing$
+    { format.bvolume output
+      new.block
+      format.number.series output
+      new.sentence
+      publisher "publisher" output.check
+      address output
+    }
+    { new.block
+      format.book.crossref output.nonnull
+    }
+  if$
+  format.edition output
+  format.date "year" output.check
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+  format.authors output
+  new.block
+  format.title "title" output.check
+  howpublished address new.block.checkb
+  howpublished output
+  address output
+  format.date output
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+  author empty$
+    { format.editors "author and editor" output.check }
+    { format.authors output.nonnull
+
+      crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+      if$
+    }
+  if$
+  new.block
+  format.btitle "title" output.check
+  crossref missing$
+    { format.bvolume output
+      format.chapter.pages "chapter and pages" output.check
+      new.block
+      format.number.series output
+      new.sentence
+      publisher "publisher" output.check
+      address output
+    }
+    { format.chapter.pages "chapter and pages" output.check
+      new.block
+      format.book.crossref output.nonnull
+    }
+  if$
+  format.edition output
+  format.date "year" output.check
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  crossref missing$
+    { format.in.ed.booktitle "booktitle" output.check
+      format.bvolume output
+      format.number.series output
+      format.chapter.pages output
+      new.sentence
+      publisher "publisher" output.check
+      address output
+      format.edition output
+      format.date "year" output.check
+    }
+    { format.incoll.inproc.crossref output.nonnull
+      format.chapter.pages output
+    }
+  if$
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  crossref missing$
+    { format.in.ed.booktitle "booktitle" output.check
+      format.bvolume output
+      format.number.series output
+      format.pages output
+      address empty$
+ { organization publisher new.sentence.checkb
+   organization output
+   publisher output
+   format.date "year" output.check
+ }
+ { address output.nonnull
+   format.date "year" output.check
+   new.sentence
+   organization output
+   publisher output
+ }
+      if$
+    }
+    { format.incoll.inproc.crossref output.nonnull
+      format.pages output
+    }
+  if$
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+  author empty$
+    { organization empty$
+ 'skip$
+ { organization output.nonnull
+   address output
+ }
+      if$
+    }
+    { format.authors output.nonnull }
+  if$
+  new.block
+  format.btitle "title" output.check
+  author empty$
+    { organization empty$
+ { address new.block.checka
+   address output
+ }
+ 'skip$
+      if$
+    }
+    { organization address new.block.checkb
+      organization output
+      address output
+    }
+  if$
+  format.edition output
+  format.date output
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  "Master's thesis" format.thesis.type output.nonnull
+  school "school" output.check
+  address output
+  format.date "year" output.check
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+  format.authors output
+  title howpublished new.block.checkb
+  format.title output
+  howpublished new.block.checka
+  howpublished output
+  format.date output
+  new.block
+  note output
+  fin.entry
+  empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.btitle "title" output.check
+  new.block
+  "PhD thesis" format.thesis.type output.nonnull
+  school "school" output.check
+  address output
+  format.date "year" output.check
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+  editor empty$
+    { organization output }
+    { format.editors output.nonnull }
+
+  if$
+  new.block
+  format.btitle "title" output.check
+  format.bvolume output
+  format.number.series output
+  address empty$
+    { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+   organization output
+ }
+      if$
+      publisher output
+      format.date "year" output.check
+    }
+    { address output.nonnull
+      format.date "year" output.check
+      new.sentence
+      editor empty$
+ 'skip$
+ { organization output }
+      if$
+      publisher output
+    }
+  if$
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  format.tr.number output.nonnull
+  institution "institution" output.check
+  address output
+  format.date "year" output.check
+  new.block
+  note output
+  fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+  format.authors "author" output.check
+  new.block
+  format.title "title" output.check
+  new.block
+  note "note" output.check
+  format.date output
+  fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"Jan."}
+
+MACRO {feb} {"Feb."}
+
+MACRO {mar} {"Mar."}
+
+MACRO {apr} {"Apr."}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"Aug."}
+
+MACRO {sep} {"Sept."}
+
+MACRO {oct} {"Oct."}
+
+MACRO {nov} {"Nov."}
+
+MACRO {dec} {"Dec."}
+
+MACRO {acmcs} {"ACM Comput. Surv."}
+
+MACRO {acta} {"Acta Inf."}
+
+MACRO {cacm} {"Commun. ACM"}
+
+MACRO {ibmjrd} {"IBM J. Res. Dev."}
+
+MACRO {ibmsj} {"IBM Syst.~J."}
+
+MACRO {ieeese} {"IEEE Trans. Softw. Eng."}
+
+MACRO {ieeetc} {"IEEE Trans. Comput."}
+
+MACRO {ieeetcad}
+ {"IEEE Trans. Comput.-Aided Design Integrated Circuits"}
+
+MACRO {ipl} {"Inf. Process. Lett."}
+
+MACRO {jacm} {"J.~ACM"}
+
+MACRO {jcss} {"J.~Comput. Syst. Sci."}
+
+MACRO {scp} {"Sci. Comput. Programming"}
+
+MACRO {sicomp} {"SIAM J. Comput."}
+
+MACRO {tocs} {"ACM Trans. Comput. Syst."}
+
+MACRO {tods} {"ACM Trans. Database Syst."}
+
+MACRO {tog} {"ACM Trans. Gr."}
+
+MACRO {toms} {"ACM Trans. Math. Softw."}
+
+MACRO {toois} {"ACM Trans. Office Inf. Syst."}
+
+MACRO {toplas} {"ACM Trans. Prog. Lang. Syst."}
+
+MACRO {tcs} {"Theoretical Comput. Sci."}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+  "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+  'len :=
+  s #1 len substring$ =
+    { s len #1 + global.max$ substring$ }
+    's
+  if$
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+  #1 'nameptr :=
+  ""
+  s num.names$ 'numnames :=
+  numnames 'namesleft :=
+    { namesleft #0 > }
+    { nameptr #1 >
+ { "   " * }
+ 'skip$
+      if$
+      s nameptr "{vv{ } }{ll{ }}{  f{ }}{  jj{ }}" format.name$ 't :=
+      nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+      if$
+      nameptr #1 + 'nameptr :=
+      namesleft #1 - 'namesleft :=
+    }
+  while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+  "A " #2
+    "An " #3
+      "The " #4 t chop.word
+    chop.word
+  chop.word
+  sortify
+  #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+    { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+   ""
+ }
+ { key sortify }
+      if$
+    }
+    { author sort.format.names }
+  if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+    { editor empty$
+ { key empty$
+     { "to sort, need author, editor, or key in " cite$ * warning$
+       ""
+     }
+     { key sortify }
+   if$
+ }
+ { editor sort.format.names }
+      if$
+    }
+    { author sort.format.names }
+  if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+
+    { organization empty$
+ { key empty$
+     { "to sort, need author, organization, or key in " cite$ * warning$
+       ""
+     }
+     { key sortify }
+   if$
+ }
+ { "The " #4 organization chop.word sortify }
+      if$
+    }
+    { author sort.format.names }
+  if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+    { organization empty$
+ { key empty$
+     { "to sort, need editor, organization, or key in " cite$ * warning$
+       ""
+     }
+     { key sortify }
+   if$
+ }
+ { "The " #4 organization chop.word sortify }
+      if$
+    }
+    { editor sort.format.names }
+  if$
+}
+
+FUNCTION {presort}
+{ type$ "book" =
+  type$ "inbook" =
+  or
+    'author.editor.sort
+    { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+     'author.organization.sort
+     'author.sort
+   if$
+ }
+      if$
+    }
+  if$
+  "    "
+  *
+  year field.or.null sortify
+  *
+  "    "
+  *
+  title field.or.null
+  sort.format.title
+  *
+  #1 entry.max$ substring$
+  'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label }
+
+INTEGERS { number.label longest.label.width }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+  #1 'number.label :=
+  #0 'longest.label.width :=
+}
+
+FUNCTION {longest.label.pass}
+{ number.label int.to.str$ 'label :=
+  number.label #1 + 'number.label :=
+  label width$ longest.label.width >
+    { label 'longest.label :=
+      label width$ 'longest.label.width :=
+    }
+    'skip$
+  if$
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {longest.label.pass}
+
+FUNCTION {begin.bib}
+{ preamble$ empty$
+    'skip$
+    { preamble$ write$ newline$ }
+  if$
+  "\begin{thebibliography}{"  longest.label  * "}" *
+  "\itemsep=-1pt" * % Compact the entries a little.
+  write$ newline$
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+  "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
+% end of file ieee.bst
+% ---------------------------------------------------------------
+
+
+

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/residual_v1_arxiv_release.bbl

@@ -0,0 +1,273 @@
+\begin{thebibliography}{10}\itemsep=-1pt
+
+\bibitem{Bengio1994}
+Y.~Bengio, P.~Simard, and P.~Frasconi.
+\newblock Learning long-term dependencies with gradient descent is difficult.
+\newblock {\em IEEE Transactions on Neural Networks}, 5(2):157--166, 1994.
+
+\bibitem{Bishop1995}
+C.~M. Bishop.
+\newblock {\em Neural networks for pattern recognition}.
+\newblock Oxford university press, 1995.
+
+\bibitem{Briggs2000}
+W.~L. Briggs, S.~F. McCormick, et~al.
+\newblock {\em {A Multigrid Tutorial}}.
+\newblock Siam, 2000.
+
+\bibitem{Chatfield2011}
+K.~Chatfield, V.~Lempitsky, A.~Vedaldi, and A.~Zisserman.
+\newblock The devil is in the details: an evaluation of recent feature encoding
+  methods.
+\newblock In {\em BMVC}, 2011.
+
+\bibitem{Everingham2010}
+M.~Everingham, L.~Van~Gool, C.~K. Williams, J.~Winn, and A.~Zisserman.
+\newblock {The Pascal Visual Object Classes (VOC) Challenge}.
+\newblock {\em IJCV}, pages 303--338, 2010.
+
+\bibitem{Gidaris2015}
+S.~Gidaris and N.~Komodakis.
+\newblock Object detection via a multi-region \& semantic segmentation-aware
+  cnn model.
+\newblock In {\em ICCV}, 2015.
+
+\bibitem{Girshick2015}
+R.~Girshick.
+\newblock {Fast R-CNN}.
+\newblock In {\em ICCV}, 2015.
+
+\bibitem{Girshick2014}
+R.~Girshick, J.~Donahue, T.~Darrell, and J.~Malik.
+\newblock Rich feature hierarchies for accurate object detection and semantic
+  segmentation.
+\newblock In {\em CVPR}, 2014.
+
+\bibitem{Glorot2010}
+X.~Glorot and Y.~Bengio.
+\newblock Understanding the difficulty of training deep feedforward neural
+  networks.
+\newblock In {\em AISTATS}, 2010.
+
+\bibitem{Goodfellow2013}
+I.~J. Goodfellow, D.~Warde-Farley, M.~Mirza, A.~Courville, and Y.~Bengio.
+\newblock Maxout networks.
+\newblock {\em arXiv:1302.4389}, 2013.
+
+\bibitem{He2015a}
+K.~He and J.~Sun.
+\newblock Convolutional neural networks at constrained time cost.
+\newblock In {\em CVPR}, 2015.
+
+\bibitem{He2014}
+K.~He, X.~Zhang, S.~Ren, and J.~Sun.
+\newblock Spatial pyramid pooling in deep convolutional networks for visual
+  recognition.
+\newblock In {\em ECCV}, 2014.
+
+\bibitem{He2015}
+K.~He, X.~Zhang, S.~Ren, and J.~Sun.
+\newblock Delving deep into rectifiers: Surpassing human-level performance on
+  imagenet classification.
+\newblock In {\em ICCV}, 2015.
+
+\bibitem{Hinton2012}
+G.~E. Hinton, N.~Srivastava, A.~Krizhevsky, I.~Sutskever, and R.~R.
+  Salakhutdinov.
+\newblock Improving neural networks by preventing co-adaptation of feature
+  detectors.
+\newblock {\em arXiv:1207.0580}, 2012.
+
+\bibitem{Hochreiter1997}
+S.~Hochreiter and J.~Schmidhuber.
+\newblock Long short-term memory.
+\newblock {\em Neural computation}, 9(8):1735--1780, 1997.
+
+\bibitem{Ioffe2015}
+S.~Ioffe and C.~Szegedy.
+\newblock Batch normalization: Accelerating deep network training by reducing
+  internal covariate shift.
+\newblock In {\em ICML}, 2015.
+
+\bibitem{Jegou2011}
+H.~Jegou, M.~Douze, and C.~Schmid.
+\newblock Product quantization for nearest neighbor search.
+\newblock {\em TPAMI}, 33, 2011.
+
+\bibitem{Jegou2012}
+H.~Jegou, F.~Perronnin, M.~Douze, J.~Sanchez, P.~Perez, and C.~Schmid.
+\newblock Aggregating local image descriptors into compact codes.
+\newblock {\em TPAMI}, 2012.
+
+\bibitem{Jia2014}
+Y.~Jia, E.~Shelhamer, J.~Donahue, S.~Karayev, J.~Long, R.~Girshick,
+  S.~Guadarrama, and T.~Darrell.
+\newblock Caffe: Convolutional architecture for fast feature embedding.
+\newblock {\em arXiv:1408.5093}, 2014.
+
+\bibitem{Krizhevsky2009}
+A.~Krizhevsky.
+\newblock Learning multiple layers of features from tiny images.
+\newblock {\em Tech Report}, 2009.
+
+\bibitem{Krizhevsky2012}
+A.~Krizhevsky, I.~Sutskever, and G.~Hinton.
+\newblock Imagenet classification with deep convolutional neural networks.
+\newblock In {\em NIPS}, 2012.
+
+\bibitem{LeCun1989}
+Y.~LeCun, B.~Boser, J.~S. Denker, D.~Henderson, R.~E. Howard, W.~Hubbard, and
+  L.~D. Jackel.
+\newblock Backpropagation applied to handwritten zip code recognition.
+\newblock {\em Neural computation}, 1989.
+
+\bibitem{LeCun1998}
+Y.~LeCun, L.~Bottou, G.~B. Orr, and K.-R. M{\"u}ller.
+\newblock Efficient backprop.
+\newblock In {\em Neural Networks: Tricks of the Trade}, pages 9--50. Springer,
+  1998.
+
+\bibitem{Lee2014}
+C.-Y. Lee, S.~Xie, P.~Gallagher, Z.~Zhang, and Z.~Tu.
+\newblock Deeply-supervised nets.
+\newblock {\em arXiv:1409.5185}, 2014.
+
+\bibitem{Lin2013}
+M.~Lin, Q.~Chen, and S.~Yan.
+\newblock Network in network.
+\newblock {\em arXiv:1312.4400}, 2013.
+
+\bibitem{Lin2014}
+T.-Y. Lin, M.~Maire, S.~Belongie, J.~Hays, P.~Perona, D.~Ramanan,
+  P.~Doll{\'a}r, and C.~L. Zitnick.
+\newblock {Microsoft COCO: Common objects in context}.
+\newblock In {\em ECCV}. 2014.
+
+\bibitem{Long2015}
+J.~Long, E.~Shelhamer, and T.~Darrell.
+\newblock Fully convolutional networks for semantic segmentation.
+\newblock In {\em CVPR}, 2015.
+
+\bibitem{Montufar2014}
+G.~Mont{\'u}far, R.~Pascanu, K.~Cho, and Y.~Bengio.
+\newblock On the number of linear regions of deep neural networks.
+\newblock In {\em NIPS}, 2014.
+
+\bibitem{Nair2010}
+V.~Nair and G.~E. Hinton.
+\newblock Rectified linear units improve restricted boltzmann machines.
+\newblock In {\em ICML}, 2010.
+
+\bibitem{Perronnin2007}
+F.~Perronnin and C.~Dance.
+\newblock Fisher kernels on visual vocabularies for image categorization.
+\newblock In {\em CVPR}, 2007.
+
+\bibitem{Raiko2012}
+T.~Raiko, H.~Valpola, and Y.~LeCun.
+\newblock Deep learning made easier by linear transformations in perceptrons.
+\newblock In {\em AISTATS}, 2012.
+
+\bibitem{Ren2015}
+S.~Ren, K.~He, R.~Girshick, and J.~Sun.
+\newblock {Faster R-CNN}: Towards real-time object detection with region
+  proposal networks.
+\newblock In {\em NIPS}, 2015.
+
+\bibitem{Ren2015a}
+S.~Ren, K.~He, R.~Girshick, X.~Zhang, and J.~Sun.
+\newblock Object detection networks on convolutional feature maps.
+\newblock {\em arXiv:1504.06066}, 2015.
+
+\bibitem{Ripley1996}
+B.~D. Ripley.
+\newblock {\em Pattern recognition and neural networks}.
+\newblock Cambridge university press, 1996.
+
+\bibitem{Romero2015}
+A.~Romero, N.~Ballas, S.~E. Kahou, A.~Chassang, C.~Gatta, and Y.~Bengio.
+\newblock Fitnets: Hints for thin deep nets.
+\newblock In {\em ICLR}, 2015.
+
+\bibitem{Russakovsky2014}
+O.~Russakovsky, J.~Deng, H.~Su, J.~Krause, S.~Satheesh, S.~Ma, Z.~Huang,
+  A.~Karpathy, A.~Khosla, M.~Bernstein, et~al.
+\newblock Imagenet large scale visual recognition challenge.
+\newblock {\em arXiv:1409.0575}, 2014.
+
+\bibitem{Saxe2013}
+A.~M. Saxe, J.~L. McClelland, and S.~Ganguli.
+\newblock Exact solutions to the nonlinear dynamics of learning in deep linear
+  neural networks.
+\newblock {\em arXiv:1312.6120}, 2013.
+
+\bibitem{Schraudolph1998a}
+N.~N. Schraudolph.
+\newblock Accelerated gradient descent by factor-centering decomposition.
+\newblock Technical report, 1998.
+
+\bibitem{Schraudolph1998}
+N.~N. Schraudolph.
+\newblock Centering neural network gradient factors.
+\newblock In {\em Neural Networks: Tricks of the Trade}, pages 207--226.
+  Springer, 1998.
+
+\bibitem{Sermanet2014}
+P.~Sermanet, D.~Eigen, X.~Zhang, M.~Mathieu, R.~Fergus, and Y.~LeCun.
+\newblock Overfeat: Integrated recognition, localization and detection using
+  convolutional networks.
+\newblock In {\em ICLR}, 2014.
+
+\bibitem{Simonyan2015}
+K.~Simonyan and A.~Zisserman.
+\newblock Very deep convolutional networks for large-scale image recognition.
+\newblock In {\em ICLR}, 2015.
+
+\bibitem{Srivastava2015}
+R.~K. Srivastava, K.~Greff, and J.~Schmidhuber.
+\newblock Highway networks.
+\newblock {\em arXiv:1505.00387}, 2015.
+
+\bibitem{Srivastava2015a}
+R.~K. Srivastava, K.~Greff, and J.~Schmidhuber.
+\newblock Training very deep networks.
+\newblock {\em 1507.06228}, 2015.
+
+\bibitem{Szegedy2015}
+C.~Szegedy, W.~Liu, Y.~Jia, P.~Sermanet, S.~Reed, D.~Anguelov, D.~Erhan,
+  V.~Vanhoucke, and A.~Rabinovich.
+\newblock Going deeper with convolutions.
+\newblock In {\em CVPR}, 2015.
+
+\bibitem{Szeliski1990}
+R.~Szeliski.
+\newblock Fast surface interpolation using hierarchical basis functions.
+\newblock {\em TPAMI}, 1990.
+
+\bibitem{Szeliski2006}
+R.~Szeliski.
+\newblock Locally adapted hierarchical basis preconditioning.
+\newblock In {\em SIGGRAPH}, 2006.
+
+\bibitem{Vatanen2013}
+T.~Vatanen, T.~Raiko, H.~Valpola, and Y.~LeCun.
+\newblock Pushing stochastic gradient towards second-order
+  methods--backpropagation learning with transformations in nonlinearities.
+\newblock In {\em Neural Information Processing}, 2013.
+
+\bibitem{Vedaldi2008}
+A.~Vedaldi and B.~Fulkerson.
+\newblock {VLFeat}: An open and portable library of computer vision algorithms,
+  2008.
+
+\bibitem{Venables1999}
+W.~Venables and B.~Ripley.
+\newblock Modern applied statistics with s-plus.
+\newblock 1999.
+
+\bibitem{Zeiler2014}
+M.~D. Zeiler and R.~Fergus.
+\newblock Visualizing and understanding convolutional neural networks.
+\newblock In {\em ECCV}, 2014.
+
+\end{thebibliography}

download/20230719163556/output/Deep-Residual-Learning-for-Image-Recognition/residual_v1_arxiv_release.tex

@@ -0,0 +1,909 @@
+\documentclass[10pt,twocolumn,letterpaper]{article}
+
+\usepackage{cvpr}
+\usepackage{times}
+\usepackage{epsfig}
+\usepackage{graphicx}
+\usepackage{amsmath}
+\usepackage{amssymb}
+\usepackage{tabulary}
+\usepackage{multirow}
+
+\usepackage{soul}
+
+% Include other packages here, before hyperref.
+
+% If you comment hyperref and then uncomment it, you should delete
+% egpaper.aux before re-running latex.  (Or just hit 'q' on the first latex
+% run, let it finish, and you should be clear).
+\usepackage[bookmarks=false,colorlinks=true,linkcolor=black,citecolor=black,filecolor=black,urlcolor=black]{hyperref}
+\usepackage[british,UKenglish,USenglish,english,american]{babel}
+
+\cvprfinalcopy % *** Uncomment this line for the final submission
+
+\newcommand{\ve}[1]{\mathbf{#1}} % for displaying a vector
+\newcommand{\ma}[1]{\mathrm{#1}} % for displaying a matrix
+
+\newcommand{\tabincell}[2]{\begin{tabular}{@{}#1@{}}#2\end{tabular}}
+
+\renewcommand\arraystretch{1.2}
+
+\def\httilde{\mbox{\tt\raisebox{-.5ex}{\symbol{126}}}}
+
+\hyphenation{identity notorious underlying surpasses desired residual doubled}
+
+
+% Pages are numbered in submission mode, and unnumbered in camera-ready
+%\ifcvprfinal\pagestyle{empty}\fi
+\setcounter{page}{1}
+\begin{document}
+
+%%%%%%%%% TITLE
+\title{Deep Residual Learning for Image Recognition}
+
+
+
+\author{Kaiming He \qquad Xiangyu Zhang \qquad Shaoqing Ren \qquad Jian Sun \\
+\large Microsoft Research \vspace{-.2em}\\
+\normalsize
+\{kahe,~v-xiangz,~v-shren,~jiansun\}@microsoft.com
+}
+
+\maketitle
+%\thispagestyle{empty}
+
+%%%%%%%%% ABSTRACT
+\begin{abstract}
+\vspace{-.5em}
+Deeper neural networks are more difficult to train. We present a residual learning framework to ease the training of networks that are substantially deeper than those used previously. We explicitly reformulate the layers as learning residual functions with reference to the layer inputs, instead of learning unreferenced functions. We provide comprehensive empirical evidence showing that these residual networks are easier to optimize, and can gain accuracy from considerably increased depth.
+On the ImageNet dataset we evaluate residual nets with a depth of up to 152 layers---8$\times$ deeper than VGG nets \cite{Simonyan2015} but still having lower complexity.
+An ensemble of these residual nets achieves 3.57\% error on the ImageNet \emph{test} set. This result won the 1st place on the ILSVRC 2015 classification task.
+We also present analysis on CIFAR-10 with 100 and 1000 layers.
+
+The depth of representations is of central importance for many visual recognition tasks. Solely due to our extremely deep representations, we obtain a 28\% relative improvement on the COCO object detection dataset. Deep residual nets are foundations of our submissions to ILSVRC \& COCO 2015 competitions\footnote{\fontsize{7.6pt}{1em}\selectfont \url{http://image-net.org/challenges/LSVRC/2015/} and \url{http://mscoco.org/dataset/\#detections-challenge2015}.}, where we also won the 1st places on the tasks of ImageNet detection, ImageNet localization, COCO detection, and COCO segmentation.
+\end{abstract}
+
+% plain text
+% Deeper neural networks are more difficult to train. We present a residual learning framework to ease the training of networks that are substantially deeper than those used previously. We explicitly reformulate the layers as learning residual functions with reference to the layer inputs, instead of learning unreferenced functions. We provide comprehensive empirical evidence showing that these residual networks are easier to optimize, and can gain accuracy from considerably increased depth. On the ImageNet dataset we evaluate residual nets with a depth of up to 152 layers---8x deeper than VGG nets but still having lower complexity. An ensemble of these residual nets achieves 3.57% error on the ImageNet test set. This result won the 1st place on the ILSVRC 2015 classification task. We also present analysis on CIFAR-10 with 100 and 1000 layers.
+
+%The depth of representations is of central importance for many visual recognition tasks. Solely due to our extremely deep representations, we obtain a 28% relative improvement on the COCO object detection dataset. Deep residual nets are foundations of our submissions to ILSVRC & COCO 2015 competitions, where we also won the 1st places on the tasks of ImageNet detection, ImageNet localization, COCO detection, and COCO segmentation.
+
+%%%%%%%%% BODY TEXT
+
+\vspace{-1em}
+\section{Introduction}
+\label{sec:intro}
+
+Deep convolutional neural networks \cite{LeCun1989,Krizhevsky2012} have led to a series of breakthroughs for image classification \cite{Krizhevsky2012,Zeiler2014,Sermanet2014}. Deep networks naturally integrate low/mid/high-level features \cite{Zeiler2014} and classifiers in an end-to-end multi-layer fashion, and the ``levels'' of features can be enriched by the number of stacked layers (depth).
+Recent evidence \cite{Simonyan2015,Szegedy2015} reveals that network depth is of crucial importance, and the leading results \cite{Simonyan2015,Szegedy2015,He2015,Ioffe2015} on the challenging ImageNet dataset \cite{Russakovsky2014} all exploit ``very deep'' \cite{Simonyan2015} models, with a depth of sixteen \cite{Simonyan2015} to thirty \cite{Ioffe2015}. Many other nontrivial visual recognition tasks \cite{Girshick2014,He2014,Girshick2015,Ren2015,Long2015} have also greatly benefited from very deep models.
+
+Driven by the significance of depth, a question arises: \emph{Is learning better networks as easy
+as stacking more layers?}
+An obstacle to answering this question was the notorious problem of vanishing/exploding gradients \cite{Bengio1994,Glorot2010}, which hamper convergence from the beginning. This problem, however, has been largely addressed by normalized initialization \cite{LeCun1998,Glorot2010,Saxe2013,He2015} and intermediate normalization layers \cite{Ioffe2015}, which enable networks with tens of layers to start converging for stochastic gradient descent (SGD) with backpropagation \cite{LeCun1989}.
+
+\begin{figure}[t]
+\begin{center}
+\includegraphics[width=1.0\linewidth]{eps/teaser}
+\end{center}
+\vspace{-1.2em}
+\caption{Training error (left) and test error (right) on CIFAR-10 with 20-layer and 56-layer ``plain'' networks. The deeper network has higher training error, and thus test error. Similar phenomena on ImageNet is presented in Fig.~\ref{fig:imagenet}.}
+\label{fig:teaser}
+\vspace{-1em}
+\end{figure}
+
+When deeper networks are able to start converging, a \emph{degradation} problem has been exposed:  with the network depth increasing, accuracy gets saturated (which might be unsurprising) and then degrades rapidly. Unexpectedly, such degradation is \emph{not caused by overfitting}, and adding more layers to a suitably deep model leads to \emph{higher training error}, as reported in \cite{He2015a, Srivastava2015} and thoroughly verified by our experiments. Fig.~\ref{fig:teaser} shows a typical example.
+
+The degradation (of training accuracy) indicates that not all systems are similarly easy to optimize. Let us consider a shallower architecture and its deeper counterpart that adds more layers onto it. There exists a solution \emph{by construction} to the deeper model: the added layers are \emph{identity} mapping, and the other layers are copied from the learned shallower model. The existence of this constructed solution indicates that a deeper model should produce no higher training error than its shallower counterpart. But experiments show that our current solvers on hand are unable to find solutions that are comparably good or better than the constructed solution (or unable to do so in feasible time).
+
+In this paper, we address the degradation problem by introducing a \emph{deep residual learning} framework.
+Instead of hoping each few stacked layers directly fit a desired underlying mapping, we explicitly let these layers fit a residual mapping. Formally, denoting the desired underlying mapping as $\mathcal{H}(\ve{x})$, we let the stacked nonlinear layers fit another mapping of $\mathcal{F}(\ve{x}):=\mathcal{H}(\ve{x})-\ve{x}$. The original mapping is recast into $\mathcal{F}(\ve{x})+\ve{x}$.
+We hypothesize that it is easier to optimize the residual mapping than to optimize the original, unreferenced mapping. To the extreme, if an identity mapping were optimal, it would be easier to push the residual to zero than to fit an identity mapping by a stack of nonlinear layers.
+
+The formulation of $\mathcal{F}(\ve{x})+\ve{x}$ can be realized by feedforward neural networks with ``shortcut connections'' (Fig.~\ref{fig:block}). Shortcut connections \cite{Bishop1995,Ripley1996,Venables1999} are those skipping one or more layers. In our case, the shortcut connections simply perform \emph{identity} mapping, and their outputs are added to the outputs of the stacked layers (Fig.~\ref{fig:block}). Identity shortcut connections add neither extra parameter nor computational complexity. The entire network can still be trained end-to-end by SGD with backpropagation, and can be easily implemented using common libraries (\eg, Caffe \cite{Jia2014}) without modifying the solvers.
+
+\begin{figure}[t]
+\centering
+\hspace{48pt}
+\includegraphics[width=0.9\linewidth]{eps/block}
+\vspace{-.5em}
+\caption{Residual learning: a building block.}
+\label{fig:block}
+\vspace{-1em}
+\end{figure}
+
+We present comprehensive experiments on ImageNet \cite{Russakovsky2014} to show the degradation problem and evaluate our method.
+We show that: 1) Our extremely deep residual nets are easy to optimize, but the counterpart ``plain'' nets (that simply stack layers) exhibit higher training error when the depth increases; 2) Our deep residual nets can easily enjoy accuracy gains from greatly increased depth, producing results substantially better than previous networks.
+
+Similar phenomena are also shown on the CIFAR-10 set \cite{Krizhevsky2009}, suggesting that the optimization difficulties and the effects of our method are not just akin to a particular dataset. We present successfully trained models on this dataset with over 100 layers, and explore models with over 1000 layers.
+
+On the ImageNet classification dataset \cite{Russakovsky2014}, we obtain excellent results by extremely deep residual nets.
+Our 152-layer residual net is the deepest network ever presented on ImageNet, while still having lower complexity than VGG nets \cite{Simonyan2015}. Our ensemble has \textbf{3.57\%} top-5 error on the ImageNet \emph{test} set, and \emph{won the 1st place in the ILSVRC 2015 classification competition}. The extremely deep representations also have excellent generalization performance on other recognition tasks, and lead us to further \emph{win the 1st places on: ImageNet detection, ImageNet localization, COCO detection, and COCO segmentation} in ILSVRC \& COCO 2015 competitions. This strong evidence shows that the residual learning principle is generic, and we expect that it is applicable in other vision and non-vision problems.
+
+
+\section{Related Work}
+
+%\vspace{6pt}
+\noindent\textbf{Residual Representations.}
+In image recognition, VLAD \cite{Jegou2012} is a representation that encodes by the residual vectors with respect to a dictionary, and Fisher Vector \cite{Perronnin2007} can be formulated as a probabilistic version \cite{Jegou2012} of VLAD.
+Both of them are powerful shallow representations for image retrieval and classification \cite{Chatfield2011,Vedaldi2008}.
+For vector quantization, encoding residual vectors \cite{Jegou2011} is shown to be more effective than encoding original vectors.
+
+In low-level vision and computer graphics, for solving Partial Differential Equations (PDEs), the widely used Multigrid method \cite{Briggs2000} reformulates the system as subproblems at multiple scales, where each subproblem is responsible for the residual solution between a coarser and a finer scale. An alternative to Multigrid is hierarchical basis preconditioning \cite{Szeliski1990,Szeliski2006}, which relies on variables that represent residual vectors between two scales. It has been shown \cite{Briggs2000,Szeliski1990,Szeliski2006} that these solvers converge much faster than standard solvers that are unaware of the residual nature of the solutions. These methods suggest that a good reformulation or preconditioning can simplify the optimization.
+
+\vspace{6pt}
+\noindent\textbf{Shortcut Connections.}
+Practices and theories that lead to shortcut connections \cite{Bishop1995,Ripley1996,Venables1999} have been studied for a long time.
+%The motivations and usages of shortcuts, however, are diversified in the literature.
+An early practice of training multi-layer perceptrons (MLPs) is to add a linear layer connected from the network input to the output \cite{Ripley1996,Venables1999}. In \cite{Szegedy2015,Lee2014}, a few intermediate layers are directly connected to auxiliary classifiers for addressing vanishing/exploding gradients. The papers of \cite{Schraudolph1998,Schraudolph1998a,Raiko2012,Vatanen2013} propose methods for centering layer responses, gradients, and propagated errors, implemented by shortcut connections. In \cite{Szegedy2015}, an ``inception'' layer is composed of a shortcut branch and a few deeper branches.
+
+
+Concurrent with our work, ``highway networks'' \cite{Srivastava2015,Srivastava2015a} present shortcut connections with gating functions \cite{Hochreiter1997}. These gates are data-dependent and have parameters, in contrast to our identity shortcuts that are parameter-free. When a gated shortcut is ``closed'' (approaching zero), the layers in highway networks represent \emph{non-residual} functions. On the contrary, our formulation always learns residual functions; our identity shortcuts are never closed, and all information is always passed through, with additional residual functions to be learned. In addition, highway networks have not demonstrated accuracy gains with extremely increased depth (\eg, over 100 layers).
+
+\section{Deep Residual Learning}
+
+\subsection{Residual Learning}
+\label{sec:motivation}
+
+Let us consider $\mathcal{H}(\ve{x})$ as an underlying mapping to be fit by a few stacked layers (not necessarily the entire net), with $\ve{x}$ denoting the inputs to the first of these layers. If one hypothesizes that multiple nonlinear layers can asymptotically approximate complicated functions\footnote{This hypothesis, however, is still an open question. See \cite{Montufar2014}.}, then it is equivalent to hypothesize that they can asymptotically approximate the residual functions, \ie, $\mathcal{H}(\ve{x})-\ve{x}$ (assuming that the input and output are of the same dimensions).
+So rather than expect stacked layers to approximate $\mathcal{H}(\ve{x})$, we explicitly let these layers approximate a residual function $\mathcal{F}(\ve{x}):=\mathcal{H}(\ve{x})-\ve{x}$. The original function thus becomes $\mathcal{F}(\ve{x})+\ve{x}$. Although both forms should be able to asymptotically approximate the desired functions (as hypothesized), the ease of learning might be different.
+
+This reformulation is motivated by the counterintuitive phenomena about the degradation problem (Fig.~\ref{fig:teaser}, left). As we discussed in the introduction, if the added layers can be constructed as identity mappings, a deeper model should have training error no greater than its shallower counterpart. The degradation problem suggests that the solvers might have difficulties in approximating identity mappings by multiple nonlinear layers. With the residual learning reformulation, if identity mappings are optimal, the solvers may simply drive the weights of the multiple nonlinear layers toward zero to approach identity mappings.
+
+In real cases, it is unlikely that identity mappings are optimal, but our reformulation may help to precondition the problem. If the optimal function is closer to an identity mapping than to a zero mapping, it should be easier for the solver to find the perturbations with reference to an identity mapping, than to learn the function as a new one. We show by experiments (Fig.~\ref{fig:std}) that the learned residual functions in general have small responses, suggesting that identity mappings provide reasonable preconditioning.
+
+\subsection{Identity Mapping by Shortcuts}
+
+We adopt residual learning to every few stacked layers.
+A building block is shown in Fig.~\ref{fig:block}. Formally, in this paper we consider a building block defined as:
+%\vspace{-.5em}
+\begin{equation}\label{eq:identity}
+\ve{y}= \mathcal{F}(\ve{x}, \{W_{i}\}) + \ve{x}.
+\end{equation}
+Here $\ve{x}$ and $\ve{y}$ are the input and output vectors of the layers considered. The function $\mathcal{F}(\ve{x}, \{W_{i}\})$ represents the residual mapping to be learned. For the example in Fig.~\ref{fig:block} that has two layers, $\mathcal{F}=W_{2}\sigma(W_{1}\ve{x})$ in which $\sigma$ denotes ReLU \cite{Nair2010} and the biases are omitted for simplifying notations. The operation $\mathcal{F}+\ve{x}$ is performed by a shortcut connection and element-wise addition. We adopt the second nonlinearity after the addition (\ie, $\sigma(\ve{y})$, see Fig.~\ref{fig:block}).
+
+The shortcut connections in Eqn.(\ref{eq:identity}) introduce neither extra parameter nor computation complexity. This is not only attractive in practice but also important in our comparisons between plain and residual networks. We can fairly compare plain/residual networks that simultaneously have the same number of parameters, depth, width, and computational cost (except for the negligible element-wise addition).
+
+The dimensions of $\ve{x}$ and $\mathcal{F}$ must be equal in Eqn.(\ref{eq:identity}). If this is not the case (\eg, when changing the input/output channels), we can perform a linear projection $W_{s}$ by the shortcut connections to match the dimensions:
+%\vspace{-.5em}
+\begin{equation}\label{eq:transform}
+\ve{y}= \mathcal{F}(\ve{x}, \{W_{i}\}) + W_{s}\ve{x}.
+\end{equation}
+We can also use a square matrix $W_{s}$ in Eqn.(\ref{eq:identity}). But we will show by experiments that the identity mapping is sufficient for addressing the degradation problem and is economical, and thus $W_{s}$ is only used when matching dimensions.
+
+The form of the residual function $\mathcal{F}$ is flexible. Experiments in this paper involve a function $\mathcal{F}$ that has two or three layers (Fig.~\ref{fig:block_deeper}), while more layers are possible. But if $\mathcal{F}$ has only a single layer, Eqn.(\ref{eq:identity}) is similar to a linear layer: $\ve{y}=W_1\ve{x}+\ve{x}$, for which we have not observed advantages.
+
+We also note that although the above notations are about fully-connected layers for simplicity, they are applicable to convolutional layers. The function $\mathcal{F}(\ve{x}, \{W_{i}\})$ can represent multiple convolutional layers. The element-wise addition is performed on two feature maps, channel by channel.
+
+\begin{figure}[t]
+\begin{center}
+\vspace{.5em}
+\includegraphics[width=1.0\linewidth]{eps/arch}
+\end{center}
+%\vspace{-1em}
+\caption{Example network architectures for ImageNet. \textbf{Left}: the VGG-19 model \cite{Simonyan2015} (19.6 billion FLOPs) as a reference. \textbf{Middle}: a plain network with 34 parameter layers (3.6 billion FLOPs). \textbf{Right}: a residual network with 34 parameter layers (3.6 billion FLOPs). The dotted shortcuts increase dimensions. \textbf{Table~\ref{tab:arch}} shows more details and other variants.}
+\label{fig:arch}
+\vspace{-1em}
+\end{figure}
+
+\subsection{Network Architectures}
+
+We have tested various plain/residual nets, and have observed consistent phenomena. To provide instances for discussion, we describe two models for ImageNet as follows.
+
+\vspace{6pt}
+\noindent\textbf{Plain Network.}
+Our plain baselines (Fig.~\ref{fig:arch}, middle) are mainly inspired by the philosophy of VGG nets \cite{Simonyan2015} (Fig.~\ref{fig:arch}, left).
+The convolutional layers mostly have 3$\times$3 filters and follow two simple design rules: (i) for the same output feature map size, the layers have the same number of filters; and (ii) if the feature map size is halved, the number of filters is doubled so as to preserve the time complexity per layer. We perform downsampling directly by convolutional layers that have a stride of 2.
+The network ends with a global average pooling layer and a 1000-way fully-connected layer with softmax. The total number of weighted layers is 34 in Fig.~\ref{fig:arch} (middle).
+
+It is worth noticing that our model has \emph{fewer} filters and \emph{lower} complexity than VGG nets \cite{Simonyan2015} (Fig.~\ref{fig:arch}, left). Our 34-layer baseline has 3.6 billion FLOPs (multiply-adds), which is only 18\% of VGG-19 (19.6 billion FLOPs).
+
+\vspace{6pt}
+\noindent\textbf{Residual Network.}
+Based on the above plain network, we insert shortcut connections (Fig.~\ref{fig:arch}, right) which turn the network into its counterpart residual version.
+The identity shortcuts (Eqn.(\ref{eq:identity})) can be directly used when the input and output are of the same dimensions (solid line shortcuts in Fig.~\ref{fig:arch}).
+When the dimensions increase (dotted line shortcuts in Fig.~\ref{fig:arch}), we consider two options:
+(A) The shortcut still performs identity mapping, with extra zero entries padded for increasing dimensions. This option introduces no extra parameter;
+(B) The projection shortcut in Eqn.(\ref{eq:transform}) is used to match dimensions (done by 1$\times$1 convolutions).
+For both options, when the shortcuts go across feature maps of two sizes, they are performed with a stride of 2.
+
+\newcommand{\blocka}[2]{\multirow{3}{*}{\(\left[\begin{array}{c}\text{3$\times$3, #1}\\[-.1em] \text{3$\times$3, #1} \end{array}\right]\)$\times$#2}
+}
+\newcommand{\blockb}[3]{\multirow{3}{*}{\(\left[\begin{array}{c}\text{1$\times$1, #2}\\[-.1em] \text{3$\times$3, #2}\\[-.1em] \text{1$\times$1, #1}\end{array}\right]\)$\times$#3}
+}
+\renewcommand\arraystretch{1.1}
+\setlength{\tabcolsep}{3pt}
+\begin{table*}[t]
+\begin{center}
+\resizebox{0.7\linewidth}{!}{
+%\footnotesize
+\begin{tabular}{c|c|c|c|c|c|c}
+\hline
+layer name & output size & 18-layer & 34-layer & 50-layer & 101-layer & 152-layer \\
+\hline
+conv1 & 112$\times$112 & \multicolumn{5}{c}{7$\times$7, 64, stride 2}\\
+\hline
+\multirow{4}{*}{conv2\_x} & \multirow{4}{*}{56$\times$56} & \multicolumn{5}{c}{3$\times$3 max pool, stride 2} \\\cline{3-7}
+  &  & \blocka{64}{2}  & \blocka{64}{3} & \blockb{256}{64}{3} & \blockb{256}{64}{3} & \blockb{256}{64}{3}\\
+  &  &  &  &  &  &\\
+  &  &  &  &  &  &\\
+\hline
+\multirow{3}{*}{conv3\_x} &  \multirow{3}{*}{28$\times$28}  & \blocka{128}{2}  & \blocka{128}{4}  & \blockb{512}{128}{4}  & \blockb{512}{128}{4}  &
+                              \blockb{512}{128}{8}\\
+  &  &  &  &  &  & \\
+  &  &  &  &  &  & \\
+\hline
+\multirow{3}{*}{conv4\_x} & \multirow{3}{*}{14$\times$14}  & \blocka{256}{2}  & \blocka{256}{6}  & \blockb{1024}{256}{6}  & \blockb{1024}{256}{23} & \blockb{1024}{256}{36}\\
+  &  &  &  &  & \\
+  &  &  &  &  & \\
+\hline
+\multirow{3}{*}{conv5\_x} & \multirow{3}{*}{7$\times$7}  & \blocka{512}{2}  & \blocka{512}{3}  & \blockb{2048}{512}{3}  & \blockb{2048}{512}{3}
+& \blockb{2048}{512}{3}\\
+  &  &  &  &  &  & \\
+  &  &  &  &  &  & \\
+\hline
+& 1$\times$1  & \multicolumn{5}{c}{average pool, 1000-d fc, softmax} \\
+\hline
+\multicolumn{2}{c|}{FLOPs} & 1.8$\times10^9$  & 3.6$\times10^9$  & 3.8$\times10^9$  & 7.6$\times10^9$  & 11.3$\times10^9$ \\
+\hline
+\end{tabular}
+}
+\end{center}
+\vspace{-.5em}
+\caption{Architectures for ImageNet. Building blocks are shown in brackets (see also Fig.~\ref{fig:block_deeper}), with the numbers of blocks stacked. Downsampling is performed by conv3\_1, conv4\_1, and conv5\_1 with a stride of 2.
+}
+\label{tab:arch}
+\vspace{-.5em}
+\end{table*}
+
+\begin{figure*}[t]
+\begin{center}
+\includegraphics[width=0.86\linewidth]{eps/imagenet}
+\end{center}
+\vspace{-1.2em}
+\caption{Training on \textbf{ImageNet}. Thin curves denote training error, and bold curves denote validation error of the center crops. Left: plain networks of 18 and 34 layers. Right: ResNets of 18 and 34 layers. In this plot, the residual networks have no extra parameter compared to their plain counterparts.}
+\label{fig:imagenet}
+\end{figure*}
+
+\subsection{Implementation}
+\label{sec:impl}
+
+Our implementation for ImageNet follows the practice in \cite{Krizhevsky2012,Simonyan2015}. The image is resized with its shorter side randomly sampled in $[256, 480]$ for scale augmentation \cite{Simonyan2015}. A 224$\times$224 crop is randomly sampled from an image or its horizontal flip, with the per-pixel mean subtracted \cite{Krizhevsky2012}. The standard color augmentation in \cite{Krizhevsky2012} is used.
+We adopt batch normalization (BN) \cite{Ioffe2015} right after each convolution and before activation, following \cite{Ioffe2015}.
+We initialize the weights as in \cite{He2015} and train all plain/residual nets from scratch.
+We use SGD with a mini-batch size of 256. The learning rate starts from 0.1 and is divided by 10 when the error plateaus, and the models are trained for up to $60\times10^4$ iterations. We use a weight decay of 0.0001 and a momentum of 0.9. We do not use dropout \cite{Hinton2012}, following the practice in \cite{Ioffe2015}.
+
+In testing, for comparison studies we adopt the standard 10-crop testing \cite{Krizhevsky2012}.
+For best results, we adopt the fully-convolutional form as in \cite{Simonyan2015,He2015}, and average the scores at multiple scales (images are resized such that the shorter side is in $\{224, 256, 384, 480, 640\}$).
+
+
+\section{Experiments}
+\label{sec:exp}
+
+\subsection{ImageNet Classification}
+\label{sec:imagenet}
+
+We evaluate our method on the ImageNet 2012 classification dataset \cite{Russakovsky2014} that consists of 1000 classes. The models are trained on the 1.28 million training images, and evaluated on the 50k validation images. We also obtain a final result on the 100k test images, reported by the test server. We evaluate both top-1 and top-5 error rates.
+
+
+\vspace{6pt}
+\noindent\textbf{Plain Networks.}
+We first evaluate 18-layer and 34-layer plain nets. The 34-layer plain net is in Fig.~\ref{fig:arch} (middle). The 18-layer plain net is of a similar form. See Table~\ref{tab:arch} for detailed architectures.
+
+The results in Table~\ref{tab:plain_vs_shortcut} show that the deeper 34-layer plain net has higher validation error than the shallower 18-layer plain net. To reveal the reasons, in Fig.~\ref{fig:imagenet} (left) we compare their training/validation errors during the training procedure. We have observed the degradation problem - the 34-layer plain net has higher \emph{training} error throughout the whole training procedure, even though the solution space of the 18-layer plain network is a subspace of that of the 34-layer one.
+
+\newcolumntype{x}[1]{>{\centering}p{#1pt}}
+\renewcommand\arraystretch{1.1}
+\setlength{\tabcolsep}{8pt}
+\begin{table}[t]
+\begin{center}
+\small
+\begin{tabular}{l|x{42}|c}
+\hline
+            & plain & ResNet \\
+\hline
+18 layers & 27.94 & 27.88 \\
+34 layers & 28.54  & \textbf{25.03}  \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Top-1 error (\%, 10-crop testing) on ImageNet validation. Here the ResNets have no extra parameter compared to their plain counterparts. Fig.~\ref{fig:imagenet} shows the training procedures.}
+\label{tab:plain_vs_shortcut}
+\end{table}
+
+We argue that this optimization difficulty is \emph{unlikely} to be caused by vanishing gradients. These plain networks are trained with BN \cite{Ioffe2015}, which ensures forward propagated signals to have non-zero variances. We also verify that the backward propagated gradients exhibit healthy norms with BN. So neither forward nor backward signals vanish.
+In fact, the 34-layer plain net is still able to achieve competitive accuracy (Table~\ref{tab:10crop}), suggesting that the solver works to some extent. We conjecture that the deep plain nets may have exponentially low convergence rates, which impact the reducing of the training error\footnote{We have experimented with more training iterations (3$\times$) and still observed the degradation problem, suggesting that this problem cannot be feasibly addressed by simply using more iterations.}.
+The reason for such optimization difficulties will be studied in the future.
+
+\vspace{6pt}
+\noindent\textbf{Residual Networks.}
+Next we evaluate 18-layer and 34-layer residual nets (\emph{ResNets}). The baseline architectures are the same as the above plain nets, expect that a shortcut connection is added to each pair of 3$\times$3 filters as in Fig.~\ref{fig:arch} (right). In the first comparison (Table~\ref{tab:plain_vs_shortcut} and Fig.~\ref{fig:imagenet} right), we use identity mapping for all shortcuts and zero-padding for increasing dimensions (option A). So they have \emph{no extra parameter} compared to the plain counterparts.
+
+We have three major observations from Table~\ref{tab:plain_vs_shortcut} and Fig.~\ref{fig:imagenet}. First, the situation is reversed with residual learning -- the 34-layer ResNet is better than the 18-layer ResNet (by 2.8\%). More importantly, the 34-layer ResNet exhibits considerably lower training error and is generalizable to the validation data. This indicates that the degradation problem is well addressed in this setting and we manage to obtain accuracy gains from increased depth.
+
+Second, compared to its plain counterpart, the 34-layer ResNet reduces the top-1 error by 3.5\% (Table~\ref{tab:plain_vs_shortcut}), resulting from the successfully reduced training error (Fig.~\ref{fig:imagenet} right \vs left). This comparison verifies the effectiveness of residual learning on extremely deep systems.
+
+Last, we also note that the 18-layer plain/residual nets are comparably accurate (Table~\ref{tab:plain_vs_shortcut}), but the 18-layer ResNet converges faster (Fig.~\ref{fig:imagenet} right \vs left).
+When the net is ``not overly deep'' (18 layers here), the current SGD solver is still able to find good solutions to the plain net. In this case, the ResNet eases the optimization by providing faster convergence at the early stage.
+
+\begin{table}[t]
+\setlength{\tabcolsep}{8pt}
+\begin{center}
+\small
+\begin{tabular}{l|cc}
+\hline
+  \footnotesize model          & \footnotesize top-1 err. & \footnotesize top-5 err. \\
+\hline
+%\footnotesize SPP-net \cite{He2014} & 29.68 & 10.95 \\
+\footnotesize VGG-16 \cite{Simonyan2015} & 28.07 & 9.33\\
+\footnotesize GoogLeNet \cite{Szegedy2015} & - & 9.15 \\
+\footnotesize PReLU-net \cite{He2015}  & 24.27 & 7.38 \\
+%\footnotesize Deep Neural Decision Forest \cite{Kontschieder2015}  & - & 7.08 \\
+\hline
+\hline
+\footnotesize plain-34 & 28.54 & 10.02 \\
+%\hline
+\footnotesize ResNet-34 A & 25.03 & 7.76 \\
+\footnotesize ResNet-34 B & 24.52 & 7.46 \\
+\footnotesize ResNet-34 C & 24.19 & 7.40 \\
+\hline
+\footnotesize ResNet-50 & 22.85 & 6.71 \\
+\footnotesize ResNet-101 & 21.75 & 6.05 \\
+\footnotesize ResNet-152 & \textbf{21.43} & \textbf{5.71} \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Error rates (\%, \textbf{10-crop} testing) on ImageNet validation.
+VGG-16 is based on our test. ResNet-50/101/152 are of option B that only uses projections for increasing dimensions.}
+\label{tab:10crop}
+\vspace{-.5em}
+\end{table}
+
+\begin{table}[t]
+\setlength{\tabcolsep}{8pt}
+\small
+\begin{center}
+\begin{tabular}{l|c c}
+\hline
+\footnotesize method & \footnotesize top-1 err. & \footnotesize top-5 err.\\
+\hline
+VGG \cite{Simonyan2015} (ILSVRC'14) & - & 8.43$^{\dag}$\\
+GoogLeNet \cite{Szegedy2015} (ILSVRC'14) & - & 7.89\\
+\hline
+VGG \cite{Simonyan2015} \footnotesize (v5) & 24.4 & 7.1\\
+PReLU-net \cite{He2015} & 21.59 & 5.71 \\
+BN-inception \cite{Ioffe2015} & 21.99 & 5.81 \\\hline
+ResNet-34 B & 21.84 & 5.71 \\
+ResNet-34 C & 21.53 & 5.60 \\
+ResNet-50 & 20.74 & 5.25 \\
+ResNet-101 & 19.87 & 4.60 \\
+ResNet-152 & \textbf{19.38} & \textbf{4.49} \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Error rates (\%) of \textbf{single-model} results on the ImageNet validation set (except $^{\dag}$ reported on the test set).}
+\label{tab:single}
+%\vspace{-.5em}
+%\end{table}
+%\begin{table}[t]
+\setlength{\tabcolsep}{12pt}
+\small
+\begin{center}
+\begin{tabular}{l|c}
+\hline
+\footnotesize method & top-5 err. (\textbf{test}) \\
+\hline
+VGG \cite{Simonyan2015} (ILSVRC'14) & 7.32\\
+GoogLeNet \cite{Szegedy2015} (ILSVRC'14) & 6.66\\
+\hline
+VGG \cite{Simonyan2015} \footnotesize (v5) & 6.8 \\
+PReLU-net \cite{He2015} & 4.94 \\
+BN-inception \cite{Ioffe2015} & 4.82 \\\hline
+\textbf{ResNet (ILSVRC'15)} & \textbf{3.57} \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Error rates (\%) of \textbf{ensembles}. The top-5 error is on the test set of ImageNet and reported by the test server.}
+\label{tab:ensemble}
+%\vspace{-.5em}
+\end{table}
+
+\vspace{6pt}
+\noindent\textbf{Identity \vs Projection Shortcuts.}
+We have shown that parameter-free, identity shortcuts help with training. Next we investigate projection shortcuts (Eqn.(\ref{eq:transform})).
+In Table~\ref{tab:10crop} we compare three options: (A) zero-padding shortcuts are used for increasing dimensions, and all shortcuts are parameter-free (the same as Table~\ref{tab:plain_vs_shortcut} and Fig.~\ref{fig:imagenet} right); (B) projection shortcuts are used for increasing dimensions, and other shortcuts are identity; and (C) all shortcuts are projections.
+
+Table~\ref{tab:10crop} shows that all three options are considerably better than the plain counterpart.
+B is slightly better than A. We argue that this is because the zero-padded dimensions in A indeed have no residual learning. C is marginally better than B, and we attribute this to the extra parameters introduced by many (thirteen) projection shortcuts. But the small differences among A/B/C indicate that projection shortcuts are not essential for addressing the degradation problem. So we do not use option C in the rest of this paper, to reduce memory/time complexity and model sizes. Identity shortcuts are particularly important for not increasing the complexity of the bottleneck architectures that are introduced below.
+
+
+\begin{figure}[t]
+\begin{center}
+\hspace{12pt}
+\includegraphics[width=0.85\linewidth]{eps/block_deeper}
+\end{center}
+%\vspace{-1.0em}
+\caption{A deeper residual function $\mathcal{F}$ for ImageNet. Left: a building block (on 56$\times$56 feature maps) as in Fig.~\ref{fig:arch} for ResNet-34. Right: a ``bottleneck'' building block for ResNet-50/101/152.}
+\label{fig:block_deeper}
+\vspace{-.6em}
+\end{figure}
+
+\vspace{6pt}
+\noindent\textbf{Deeper Bottleneck Architectures.} Next we describe our deeper nets for ImageNet. Because of concerns on the training time that we can afford, we modify the building block as a \emph{bottleneck} design\footnote{Deeper \emph{non}-bottleneck ResNets (\eg, Fig.~\ref{fig:block_deeper} left) also gain accuracy from increased depth (as shown on CIFAR-10), but are not as economical as the bottleneck ResNets. So the usage of bottleneck designs is mainly due to practical considerations. We further note that the degradation problem of plain nets is also witnessed for the bottleneck designs.}.
+For each residual function $\mathcal{F}$, we use a stack of 3 layers instead of 2 (Fig.~\ref{fig:block_deeper}). The three layers are 1$\times$1, 3$\times$3, and 1$\times$1 convolutions, where the 1$\times$1 layers are responsible for reducing and then increasing (restoring) dimensions, leaving the 3$\times$3 layer a bottleneck with smaller input/output dimensions.
+Fig.~\ref{fig:block_deeper} shows an example, where both designs have similar time complexity.
+
+The parameter-free identity shortcuts are particularly important for the bottleneck architectures. If the identity shortcut in Fig.~\ref{fig:block_deeper} (right) is replaced with projection, one can show that the time complexity and model size are doubled, as the shortcut is connected to the two high-dimensional ends. So identity shortcuts lead to more efficient models for the bottleneck designs.
+
+\textbf{50-layer ResNet:} We replace each 2-layer block in the 34-layer net with this 3-layer bottleneck block, resulting in a 50-layer ResNet (Table~\ref{tab:arch}). We use option B for increasing dimensions.
+This model has 3.8 billion FLOPs.
+
+\textbf{101-layer and 152-layer ResNets:} We construct 101-layer and 152-layer ResNets by using more 3-layer blocks (Table~\ref{tab:arch}).
+Remarkably, although the depth is significantly increased, the 152-layer ResNet (11.3 billion FLOPs) still has \emph{lower complexity} than VGG-16/19 nets (15.3/19.6 billion FLOPs).
+
+The 50/101/152-layer ResNets are more accurate than the 34-layer ones by considerable margins (Table~\ref{tab:10crop} and~\ref{tab:single}). We do not observe the degradation problem and thus enjoy significant accuracy gains from considerably increased depth. The benefits of depth are witnessed for all evaluation metrics (Table~\ref{tab:10crop} and~\ref{tab:single}).
+
+\begin{figure*}[t]
+\begin{center}
+\includegraphics[width=0.8\linewidth]{eps/cifar}
+\end{center}
+\vspace{-1.5em}
+\caption{Training on \textbf{CIFAR-10}. Dashed lines denote training error, and bold lines denote testing error. \textbf{Left}: plain networks. The error of plain-110 is higher than 60\% and not displayed. \textbf{Middle}: ResNets. \textbf{Right}: ResNets with 110 and 1202 layers.}
+\label{fig:cifar}
+\end{figure*}
+
+\vspace{6pt}
+\noindent\textbf{Comparisons with State-of-the-art Methods.}
+In Table~\ref{tab:single} we compare with the previous best single-model results.
+Our baseline 34-layer ResNets have achieved very competitive accuracy.
+Our 152-layer ResNet has a single-model top-5 validation error of 4.49\%. This single-model result outperforms all previous ensemble results (Table~\ref{tab:ensemble}).
+We combine six models of different depth to form an ensemble (only with two 152-layer ones at the time of submitting). This leads to \textbf{3.57\%} top-5 error on the test set (Table~\ref{tab:ensemble}). \emph{This entry won the 1st place in ILSVRC 2015.}
+
+
+\subsection{CIFAR-10 and Analysis}
+
+We conducted more studies on the CIFAR-10 dataset \cite{Krizhevsky2009}, which consists of 50k training images and 10k testing images in 10 classes. We present experiments trained on the training set and evaluated on the test set.
+Our focus is on the behaviors of extremely deep networks, but not on pushing the state-of-the-art results, so we intentionally use simple architectures as follows.
+
+The plain/residual architectures follow the form in Fig.~\ref{fig:arch} (middle/right).
+The network inputs are 32$\times$32 images, with the per-pixel mean subtracted. The first layer is 3$\times$3 convolutions. Then we use a stack of $6n$ layers with 3$\times$3 convolutions on the feature maps of sizes $\{32, 16, 8\}$ respectively, with 2$n$ layers for each feature map size. The numbers of filters are $\{16, 32, 64\}$ respectively. The subsampling is performed by convolutions with a stride of 2. The network ends with a global average pooling, a 10-way fully-connected layer, and softmax. There are totally 6$n$+2 stacked weighted layers. The following table summarizes the architecture:
+\renewcommand\arraystretch{1.1}
+\begin{center}
+%\footnotesize
+\small
+\setlength{\tabcolsep}{8pt}
+\begin{tabular}{c|c|c|c}
+\hline
+output map size & 32$\times$32 & 16$\times$16 & 8$\times$8 \\
+\hline
+\# layers & 1+2$n$ & 2$n$ & 2$n$\\
+\# filters & 16 & 32 & 64\\
+\hline
+\end{tabular}
+\end{center}
+When shortcut connections are used, they are connected to the pairs of 3$\times$3 layers (totally $3n$ shortcuts). On this dataset we use identity shortcuts in all cases (\ie, option A), so our residual models have exactly the same depth, width, and number of parameters as the plain counterparts.
+
+\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{5pt}
+\begin{table}[t]
+\begin{center}
+\small
+\resizebox{1.0\linewidth}{!}{
+\begin{tabular}{c|c|c|l}
+\hline
+  \multicolumn{3}{c|}{method}   & error (\%) \\
+\hline
+\multicolumn{3}{c|}{Maxout \cite{Goodfellow2013}} & 9.38 \\
+\multicolumn{3}{c|}{NIN \cite{Lin2013}} & 8.81 \\
+\multicolumn{3}{c|}{DSN \cite{Lee2014}} & 8.22 \\
+\hline
+ & \# layers & \# params &  \\
+\hline
+FitNet \cite{Romero2015} & 19 & 2.5M  & 8.39 \\
+Highway \cite{Srivastava2015,Srivastava2015a} & 19 & 2.3M  & 7.54 \footnotesize (7.72$\pm$0.16) \\
+Highway \cite{Srivastava2015,Srivastava2015a} & 32 & 1.25M  & 8.80 \\
+\hline
+ResNet & 20 & 0.27M & 8.75 \\
+ResNet & 32 & 0.46M & 7.51 \\
+ResNet & 44 & 0.66M & 7.17 \\
+ResNet & 56 & 0.85M & 6.97 \\
+ResNet & 110 & 1.7M & \textbf{6.43} \footnotesize (6.61$\pm$0.16) \\ % 6.64, 6.43, 6.61, 6.53, 6.86
+ResNet & 1202 & 19.4M & 7.93 \\
+\hline
+\end{tabular}
+}
+\end{center}
+\caption{Classification error on the \textbf{CIFAR-10} test set. All methods are with data augmentation. For ResNet-110, we run it 5 times and show ``best (mean$\pm$std)'' as in \cite{Srivastava2015a}.%, and our 5 results are \{6.64, 6.43, 6.61, 6.53, 6.86\}.
+}
+\label{tab:cifar}
+\end{table}
+
+We use a weight decay of 0.0001 and momentum of 0.9, and adopt the weight initialization in \cite{He2015} and BN \cite{Ioffe2015} but with no dropout. These models are trained with a mini-batch size of 128 on two GPUs. We start with a learning rate of 0.1, divide it by 10 at 32k and 48k iterations, and terminate training at 64k iterations, which is determined on a 45k/5k train/val split. We follow the simple data augmentation in \cite{Lee2014} for training: 4 pixels are padded on each side, and a 32$\times$32 crop is randomly sampled from the padded image or its horizontal flip. For testing, we only evaluate the single view of the original 32$\times$32 image.
+
+We compare $n=\{3,5,7,9\}$, leading to 20, 32, 44, and 56-layer networks.
+Fig.~\ref{fig:cifar} (left) shows the behaviors of the plain nets. The deep plain nets suffer from increased depth, and exhibit higher training error when going deeper. This phenomenon is similar to that on ImageNet (Fig.~\ref{fig:imagenet}, left) and on MNIST (see \cite{Srivastava2015}), suggesting that such an optimization difficulty is a fundamental problem.
+
+Fig.~\ref{fig:cifar} (middle) shows the behaviors of ResNets. Also similar to the ImageNet cases (Fig.~\ref{fig:imagenet}, right), our ResNets manage to overcome the optimization difficulty and demonstrate accuracy gains when the depth increases.
+
+We further explore $n=18$ that leads to a 110-layer ResNet. In this case, we find that the initial learning rate of 0.1 is slightly too large to start converging\footnote{With an initial learning rate of 0.1, it starts converging ($<$90\% error) after several epochs, but still reaches similar accuracy.}. So we use 0.01 to warm up the training until the training error is below 80\% (about 400 iterations), and then go back to 0.1 and continue training. The rest of the learning schedule is as done previously. This 110-layer network converges well (Fig.~\ref{fig:cifar}, middle). It has \emph{fewer} parameters than other deep and thin networks such as FitNet \cite{Romero2015} and Highway \cite{Srivastava2015} (Table~\ref{tab:cifar}), yet is among the state-of-the-art results (6.43\%, Table~\ref{tab:cifar}).
+
+\begin{figure}[t]
+\begin{center}
+\includegraphics[width=0.9\linewidth]{eps/std}
+\end{center}
+\vspace{-1.5em}
+\caption{Standard deviations (std) of layer responses on CIFAR-10. The responses are the outputs of each 3$\times$3 layer, after BN and before nonlinearity. \textbf{Top}: the layers are shown in their original order. \textbf{Bottom}: the responses are ranked in descending order.}
+\label{fig:std}
+\end{figure}
+
+
+\vspace{6pt}
+\noindent\textbf{Analysis of Layer Responses.}
+Fig.~\ref{fig:std} shows the standard deviations (std) of the layer responses. The responses are the outputs of each 3$\times$3 layer, after BN and before other nonlinearity (ReLU/addition). For ResNets, this analysis reveals the response strength of the residual functions.
+Fig.~\ref{fig:std} shows that ResNets have generally smaller responses than their plain counterparts. These results support our basic motivation (Sec.\ref{sec:motivation}) that the residual functions might be generally closer to zero than the non-residual functions.
+We also notice that the deeper ResNet has smaller magnitudes of responses, as evidenced by the comparisons among ResNet-20, 56, and 110 in Fig.~\ref{fig:std}. When there are more layers, an individual layer of ResNets tends to modify the signal less.
+
+\vspace{6pt}
+\noindent\textbf{Exploring Over 1000 layers.}
+We explore an aggressively deep model of over 1000 layers. We set $n=200$ that leads to a 1202-layer network, which is trained as described above. Our method shows \emph{no optimization difficulty}, and this $10^3$-layer network is able to achieve \emph{training error} $<$0.1\% (Fig.~\ref{fig:cifar}, right). Its test error is still fairly good (7.93\%, Table~\ref{tab:cifar}).
+
+But there are still open problems on such aggressively deep models.
+The testing result of this 1202-layer network is worse than that of our 110-layer network, although both have similar training error. We argue that this is because of overfitting.
+The 1202-layer network may be unnecessarily large (19.4M) for this small dataset. Strong regularization such as maxout \cite{Goodfellow2013} or dropout \cite{Hinton2012} is applied to obtain the best results (\cite{Goodfellow2013,Lin2013,Lee2014,Romero2015}) on this dataset.
+In this paper, we use no maxout/dropout and just simply impose regularization via deep and thin architectures by design, without distracting from the focus on the difficulties of optimization. But combining with stronger regularization may improve results, which we will study in the future.
+
+\subsection{Object Detection on PASCAL and MS COCO}
+
+\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{8pt}
+\begin{table}[t]
+\begin{center}
+\small
+%\resizebox{0.9\linewidth}{!}{
+\begin{tabular}{c|c|c}
+\hline
+training data & 07+12 & 07++12 \\
+\hline
+test data & VOC 07 test & VOC 12 test \\
+\hline
+VGG-16 & 73.2 & 70.4 \\
+ResNet-101 & \textbf{76.4} & \textbf{73.8} \\
+\hline
+\end{tabular}
+%}
+\end{center}
+\vspace{-.5em}
+\caption{Object detection mAP (\%) on the PASCAL VOC 2007/2012 test sets using \textbf{baseline} Faster R-CNN. See also Table~\ref{tab:voc07_all} and \ref{tab:voc12_all} for better results.
+}
+\vspace{-.5em}
+\label{tab:detection_voc}
+%\end{table}
+%\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{5pt}
+%\begin{table}[t]
+\begin{center}
+\small
+%\resizebox{0.9\linewidth}{!}{
+\begin{tabular}{c|c|c}
+\hline
+metric & [email protected]~~~ & mAP@[.5, .95] \\
+\hline
+VGG-16 & 41.5 & 21.2 \\
+ResNet-101 & \textbf{48.4} & \textbf{27.2} \\
+\hline
+\end{tabular}
+%}
+\end{center}
+\vspace{-.5em}
+\caption{Object detection mAP (\%) on the COCO validation set using \textbf{baseline} Faster R-CNN. See also Table~\ref{tab:detection_coco_improve} for better results.
+}
+\vspace{-.5em}
+\label{tab:detection_coco}
+\end{table}
+
+Our method has good generalization performance on other recognition tasks. Table~\ref{tab:detection_voc} and ~\ref{tab:detection_coco} show the object detection baseline results on PASCAL VOC 2007 and 2012 \cite{Everingham2010} and COCO \cite{Lin2014}. We adopt \emph{Faster R-CNN} \cite{Ren2015} as the detection method. Here we are interested in the improvements of replacing VGG-16 \cite{Simonyan2015} with ResNet-101. The detection implementation (see appendix) of using both models is the same, so the gains can only be attributed to better networks. Most remarkably, on the challenging COCO dataset we obtain a 6.0\% increase in COCO's standard metric (mAP@[.5, .95]), which is a 28\% relative improvement. This gain is solely due to the learned representations.
+
+Based on deep residual nets, we won the 1st places in several tracks in ILSVRC \& COCO 2015 competitions: ImageNet detection, ImageNet localization, COCO detection, and COCO segmentation. The details are in the appendix.
+
+
+
+
+
+%\renewcommand\arraystretch{1.05}
+%\setlength{\tabcolsep}{4pt}
+%\begin{table}[t]
+%\begin{center}
+%\small
+%\resizebox{1.0\linewidth}{!}{
+%\begin{tabular}{c|c|c|c|c}
+%\hline
+%dataset & VOC 07 & VOC 12 & \multicolumn{2}{c}{MS COCO} \\
+%\hline
+%metric & [email protected] & [email protected] & [email protected] & mAP@[.5, .95] \\
+%\hline
+%VGG-16 & 73.2 & 70.4 & 41.5 & 21.2 \\
+%ResNet-101 & \textbf{76.4} & \textbf{73.8} & \textbf{48.4} & \textbf{27.2} \\
+%\hline
+%\end{tabular}
+%}
+%\end{center}
+%\vspace{-.5em}
+%\caption{\textbf{Object detection} mAP (\%) on the PASCAL VOC 2007/2012 test sets and the MS COCO validation set using Faster R-CNN \cite{Ren2015}. Following \cite{Ren2015}, for the VOC 2007 test, the training data is 2007 trainval and 2012 trainval; for the VOC 2012 test, the training data is 2007 trainval+test and 2012 trainval. For MS COCO, the training data is the 80k COCO training set. ``[email protected]'' means that IoU=0.5 is used for evaluation.
+%}
+%\label{tab:detection}
+%\end{table}
+
+{
+%\small
+\footnotesize
+%\fontsize{8.8pt}{9pt}\selectfont
+\bibliographystyle{ieee}
+\bibliography{residual_v1_arxiv_release}
+}
+
+\newpage
+
+\appendix
+\section{Object Detection Baselines}
+
+In this section we introduce our detection method based on the baseline Faster R-CNN \cite{Ren2015} system.
+The models are initialized by the ImageNet classification models, and then fine-tuned on the object detection data. We have experimented with ResNet-50/101 at the time of the ILSVRC \& COCO 2015 detection competitions.
+
+Unlike VGG-16 used in \cite{Ren2015}, our ResNet has no hidden fc layers. We adopt the idea of ``Networks on Conv feature maps'' (NoC) \cite{Ren2015a} to address this issue.
+We compute the full-image shared conv feature maps using those layers whose strides on the image are no greater than 16 pixels (\ie, conv1, conv2\_ x, conv3\_x, and conv4\_x, totally 91 conv layers in ResNet-101; Table~\ref{tab:arch}). We consider these layers as analogous to the 13 conv layers in VGG-16, and by doing so, both ResNet and VGG-16 have conv feature maps of the same total stride (16 pixels).
+These layers are shared by a region proposal network (RPN, generating 300 proposals) \cite{Ren2015} and a Fast R-CNN detection network \cite{Girshick2015}.
+RoI pooling \cite{Girshick2015} is performed before conv5\_1. On this RoI-pooled feature, all layers of conv5\_x and up are adopted for each region, playing the roles of VGG-16's fc layers.
+The final classification layer is replaced by two sibling layers (classification and box regression \cite{Girshick2015}).
+
+For the usage of BN layers, after pre-training, we compute the BN statistics (means and variances) for each layer on the ImageNet training set. Then the BN layers are fixed during fine-tuning for object detection. As such, the BN layers become linear activations with constant offsets and scales, and BN statistics are not updated by fine-tuning. We fix the BN layers mainly for reducing memory consumption in Faster R-CNN training.
+
+
+\vspace{.5em}
+\noindent\textbf{PASCAL VOC}
+
+Following \cite{Girshick2015,Ren2015}, for the PASCAL VOC 2007 \emph{test} set, we use the 5k \emph{trainval} images in VOC 2007 and 16k \emph{trainval} images in VOC 2012 for training (``07+12''). For the PASCAL VOC 2012 \emph{test} set, we use the 10k \emph{trainval}+\emph{test} images in VOC 2007 and 16k \emph{trainval} images in VOC 2012 for training (``07++12''). The hyper-parameters for training Faster R-CNN are the same as in \cite{Ren2015}.
+Table~\ref{tab:detection_voc} shows the results. ResNet-101 improves the mAP by $>$3\% over VGG-16. This gain is solely because of the improved features learned by ResNet.
+
+
+
+\vspace{.5em}
+\noindent\textbf{MS COCO}
+
+The MS COCO dataset \cite{Lin2014} involves 80 object categories. We evaluate the PASCAL VOC metric (mAP @ IoU = 0.5) and the standard COCO metric (mAP @ IoU = .5:.05:.95). We use the 80k images on the train set for training and the 40k images on the val set for evaluation.
+Our detection system for COCO is similar to that for PASCAL VOC.
+We train the COCO models with an 8-GPU implementation, and thus the RPN step has a mini-batch size of 8 images (\ie, 1 per GPU) and the Fast R-CNN step has a mini-batch size of 16 images. The RPN step and Fast R-CNN step are both trained for 240k iterations with a learning rate of 0.001 and then for 80k iterations with 0.0001.
+
+
+Table~\ref{tab:detection_coco} shows the results on the MS COCO validation set. ResNet-101 has a 6\% increase of mAP@[.5, .95] over VGG-16, which is a 28\% relative improvement, solely contributed by the features learned by the better network. Remarkably, the mAP@[.5, .95]'s absolute increase (6.0\%) is nearly as big as [email protected]'s (6.9\%). This suggests that a deeper network can improve both recognition and localization.
+
+
+
+\section{Object Detection Improvements}
+
+For completeness, we report the improvements made for the competitions. These improvements are based on deep features and thus should benefit from residual learning.
+
+\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{4pt}
+\begin{table*}[t]
+\begin{center}
+\small
+%\resizebox{0.9\linewidth}{!}{
+\begin{tabular}{l|c|c|c|c}
+\hline
+training data & \multicolumn{2}{c|}{COCO train} & \multicolumn{2}{c}{COCO trainval} \\
+\hline
+test data & \multicolumn{2}{c|}{COCO val} & \multicolumn{2}{c}{COCO test-dev}\\
+\hline
+mAP & [email protected]~~~~ & @[.5, .95] & [email protected]~~~~ & @[.5, .95]\\
+\hline
+baseline Faster R-CNN (VGG-16)    & 41.5 & 21.2 & \\
+baseline Faster R-CNN (ResNet-101)    & 48.4 & 27.2 & \\
+~+box refinement &  49.9 & 29.9 & \\
+~+context &  51.1 & 30.0 & 53.3 & 32.2 \\
+~+multi-scale testing & 53.8 & 32.5 & \textbf{55.7} & \textbf{34.9} \\
+\hline
+ensemble & & & \textbf{59.0} & \textbf{37.4} \\
+\hline
+\end{tabular}
+%}
+\end{center}
+\vspace{-.5em}
+\caption{Object detection improvements on MS COCO using Faster R-CNN and ResNet-101.}
+\vspace{-.5em}
+\label{tab:detection_coco_improve}
+\end{table*}
+
+\newcolumntype{x}[1]{>{\centering}p{#1pt}}
+\newcolumntype{y}{>{\centering}p{16pt}}
+\renewcommand{\hl}[1]{\textbf{#1}}
+\newcommand{\ct}[1]{\fontsize{6pt}{1pt}\selectfont{#1}}
+\renewcommand{\arraystretch}{1.2}
+\setlength{\tabcolsep}{1.5pt}
+\begin{table*}[t]
+\begin{center}
+\footnotesize
+\vspace{1em}
+\resizebox{\linewidth}{!}{
+\begin{tabular}{l|x{40}|x{54}|x{20}|yyyyyyyyyyyyyyyyyyyc}
+\hline
+\ct{system} & net & data & mAP & \ct{areo} & \ct{bike} & \ct{bird} & \ct{boat} & \ct{bottle} & \ct{bus} & \ct{car} & \ct{cat} & \ct{chair} & \ct{cow} & \ct{table} & \ct{dog} & \ct{horse} & \ct{mbike} & \ct{person} & \ct{plant} & \ct{sheep} & \ct{sofa} & \ct{train} & \ct{tv} \\
+\hline
+\footnotesize baseline & \footnotesize VGG-16 & 07+12 & {73.2} & 76.5 & 79.0 & {70.9} & {65.5} & {52.1} & {83.1} & {84.7} & 86.4 & 52.0 & {81.9} & 65.7 & {84.8} & {84.6} & {77.5} & {76.7} & 38.8 & {73.6} & 73.9 & {83.0} & {72.6}\\
+\footnotesize baseline & \footnotesize ResNet-101 & 07+12 & 76.4 & 79.8 & 80.7 & 76.2 & 68.3 & 55.9 & 85.1 & 85.3 & \hl{89.8} & 56.7 & 87.8 & 69.4 & 88.3 & 88.9 & 80.9 & 78.4 & 41.7 & 78.6 & 79.8 & 85.3 & 72.0 \\
+\footnotesize baseline+++ & \footnotesize ResNet-101 & COCO+07+12 & \hl{85.6} & \hl{90.0} & \hl{89.6} & \hl{87.8} & \hl{80.8} & \hl{76.1} & \hl{89.9} & \hl{89.9} & {89.6} & \hl{75.5} & \hl{90.0} & \hl{80.7} & \hl{89.6} & \hl{90.3} & \hl{89.1} & \hl{88.7} & \hl{65.4} & \hl{88.1} & \hl{85.6} & \hl{89.0} & \hl{86.8} \\
+\hline
+\end{tabular}
+}
+\end{center}
+\vspace{-.5em}
+\caption{Detection results on the PASCAL VOC 2007 test set. The baseline is the Faster R-CNN system. The system ``baseline+++'' include box refinement, context, and multi-scale testing in Table~\ref{tab:detection_coco_improve}.}
+\label{tab:voc07_all}
+%\end{table*}
+%\begin{table*}[t]
+\begin{center}
+\footnotesize
+%\vspace{1em}
+\resizebox{\linewidth}{!}{
+\begin{tabular}{l|x{40}|x{54}|x{20}|yyyyyyyyyyyyyyyyyyyc}
+\hline
+\ct{system} & net & data & mAP & \ct{areo} & \ct{bike} & \ct{bird} & \ct{boat} & \ct{bottle} & \ct{bus} & \ct{car} & \ct{cat} & \ct{chair} & \ct{cow} & \ct{table} & \ct{dog} & \ct{horse} & \ct{mbike} & \ct{person} & \ct{plant} & \ct{sheep} & \ct{sofa} & \ct{train} & \ct{tv} \\
+\hline
+\footnotesize baseline & \footnotesize VGG-16 & 07++12 & {70.4} & {84.9} & {79.8} & {74.3} & {53.9} & {49.8} & 77.5 & {75.9} & 88.5 & {45.6} & {77.1} & {55.3} & 86.9 & {81.7} & {80.9} & {79.6} & {40.1} & {72.6} & 60.9 & {81.2} & 61.5\\
+\footnotesize baseline & \footnotesize ResNet-101 & 07++12 & 73.8 & 86.5 & 81.6 & 77.2 & 58.0 & 51.0 & 78.6 & 76.6 & 93.2 & 48.6 & 80.4 & 59.0 & 92.1 & 85.3 & 84.8 & 80.7 & 48.1 & 77.3 & 66.5 & 84.7 & 65.6 \\
+\footnotesize baseline+++ & \footnotesize ResNet-101 & COCO+07++12 & \hl{83.8} & \hl{92.1} & \hl{88.4} & \hl{84.8} & \hl{75.9} & \hl{71.4} & \hl{86.3} & \hl{87.8} & \hl{94.2} & \hl{66.8} & \hl{89.4} & \hl{69.2} & \hl{93.9} & \hl{91.9} & \hl{90.9} & \hl{ 89.6} & \hl{67.9} & \hl{88.2} & \hl{76.8} & \hl{90.3} & \hl{80.0} \\
+\hline
+\end{tabular}
+}
+\end{center}
+\vspace{-.5em}
+\caption{Detection results on the PASCAL VOC 2012 test set (\url{http://host.robots.ox.ac.uk:8080/leaderboard/displaylb.php?challengeid=11&compid=4}). The baseline is the Faster R-CNN system. The system ``baseline+++'' include box refinement, context, and multi-scale testing in Table~\ref{tab:detection_coco_improve}.}
+\label{tab:voc12_all}
+\end{table*}
+
+\vspace{.5em}
+\noindent\textbf{MS COCO}
+
+%\vspace{.5em}
+\noindent\emph{Box refinement.} Our box refinement partially follows the iterative localization in \cite{Gidaris2015}.
+In Faster R-CNN, the final output is a regressed box that is different from its proposal box. So for inference, we pool a new feature from the regressed box and obtain a new classification score and a new regressed box.
+We combine these 300 new predictions with the original 300 predictions. Non-maximum suppression (NMS) is applied on the union set of predicted boxes using an IoU threshold of 0.3 \cite{Girshick2014}, followed by box voting \cite{Gidaris2015}.
+Box refinement improves mAP by about 2 points (Table~\ref{tab:detection_coco_improve}).
+
+\vspace{.5em}
+\noindent\emph{Global context.} We combine global context in the Fast R-CNN step. Given the full-image conv feature map, we pool a feature by global Spatial Pyramid Pooling \cite{He2014} (with a ``single-level'' pyramid) which can be implemented as ``RoI'' pooling using the entire image's bounding box as the RoI. This pooled feature is fed into the post-RoI layers to obtain a global context feature. This global feature is concatenated with the original per-region feature, followed by the sibling classification and box regression layers. This new structure is trained end-to-end.
+Global context improves [email protected] by about 1 point (Table~\ref{tab:detection_coco_improve}).
+
+\vspace{.5em}
+\noindent\emph{Multi-scale testing.} In the above, all results are obtained by single-scale training/testing as in \cite{Ren2015}, where the image's shorter side is $s=600$ pixels. Multi-scale training/testing has been developed in \cite{He2014,Girshick2015} by selecting a scale from a feature pyramid, and in \cite{Ren2015a} by using maxout layers. In our current implementation, we have performed multi-scale \emph{testing} following \cite{Ren2015a}; we have not performed multi-scale training because of limited time. In addition, we have performed multi-scale testing only for the Fast R-CNN step (but not yet for the RPN step).
+With a trained model, we compute conv feature maps on an image pyramid, where the image's shorter sides are $s\in\{200, 400, 600, 800, 1000\}$. We select two adjacent scales from the pyramid following \cite{Ren2015a}. RoI pooling and subsequent layers are performed on the feature maps of these two scales \cite{Ren2015a}, which are merged by maxout as in \cite{Ren2015a}.
+Multi-scale testing improves the mAP by over 2 points (Table~\ref{tab:detection_coco_improve}).
+
+\vspace{.5em}
+\noindent\emph{Using validation data.} Next we use the 80k+40k trainval set for training and the 20k test-dev set for evaluation. The test-dev set has no publicly available ground truth and the result is reported by the evaluation server. Under this setting, the results are an [email protected] of 55.7\% and an mAP@[.5, .95] of 34.9\% (Table~\ref{tab:detection_coco_improve}). This is our single-model result.
+
+\vspace{.5em}
+\noindent\emph{Ensemble.} In Faster R-CNN, the system is designed to learn region proposals and also object classifiers, so an ensemble can be used to boost both tasks. We use an ensemble for proposing regions, and the union set of proposals are processed by an ensemble of per-region classifiers.
+Table~\ref{tab:detection_coco_improve} shows our result based on an ensemble of 3 networks. The mAP is 59.0\% and 37.4\% on the test-dev set. \emph{This result won the 1st place in the detection task in COCO 2015.}
+
+
+\vspace{1em}
+\noindent\textbf{PASCAL VOC}
+
+We revisit the PASCAL VOC dataset based on the above model. With the single model on the COCO dataset (55.7\% [email protected] in Table~\ref{tab:detection_coco_improve}), we fine-tune this model on the PASCAL VOC sets. The improvements of box refinement, context, and multi-scale testing are also adopted. By doing so we achieve 85.6\% mAP on PASCAL VOC 2007 (Table~\ref{tab:voc07_all}) and 83.8\% on PASCAL VOC 2012 (Table~\ref{tab:voc12_all})\footnote{\fontsize{6.5pt}{1em}\selectfont\url{http://host.robots.ox.ac.uk:8080/anonymous/3OJ4OJ.html}, submitted on 2015-11-26.}. The result on PASCAL VOC 2012 is 10 points higher than the previous state-of-the-art result \cite{Gidaris2015}.
+
+
+\vspace{1em}
+\noindent\textbf{ImageNet Detection}
+
+\renewcommand\arraystretch{1.2}
+\setlength{\tabcolsep}{10pt}
+\begin{table}[t]
+\begin{center}
+\small
+\begin{tabular}{l|c|c}
+\hline
+  & val2 & test \\
+\hline
+GoogLeNet \cite{Szegedy2015} (ILSVRC'14) & - & 43.9 \\
+\hline
+our single model (ILSVRC'15) & 60.5 & 58.8  \\
+our ensemble (ILSVRC'15) & \textbf{63.6} & \textbf{62.1} \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Our results (mAP, \%) on the ImageNet detection dataset. Our detection system is Faster R-CNN \cite{Ren2015} with the improvements in Table~\ref{tab:detection_coco_improve}, using ResNet-101.
+}
+\vspace{-.5em}
+\label{tab:imagenet_det}
+\end{table}
+
+The ImageNet Detection (DET) task involves 200 object categories. The accuracy is evaluated by [email protected].
+Our object detection algorithm for ImageNet DET is the same as that for MS COCO in Table~\ref{tab:detection_coco_improve}. The networks are pre-trained on the 1000-class ImageNet classification set, and are fine-tuned on the DET data. We split the validation set into two parts (val1/val2) following \cite{Girshick2014}. We fine-tune the detection models using the DET training set and the val1 set. The val2 set is used for validation. We do not use other ILSVRC 2015 data. Our single model with ResNet-101 has 58.8\% mAP and our ensemble of 3 models has 62.1\% mAP on the DET test set (Table~\ref{tab:imagenet_det}). \emph{This result won the 1st place in the ImageNet detection task in ILSVRC 2015}, surpassing the second place by \textbf{8.5 points} (absolute).
+
+%\end{document}
+
+\section{ImageNet Localization}
+\label{sec:appendix_localization}
+
+\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{2pt}
+\begin{table}[t]
+\begin{center}
+\small
+\resizebox{1.0\linewidth}{!}{
+\begin{tabular}{c|c|c|c|c|c}
+\hline
+\tabincell{c}{LOC \\ method} & \tabincell{c}{LOC \\ network} & testing & \tabincell{c}{LOC error \\on GT CLS} & \tabincell{c}{classification\\ network} & \tabincell{c}{top-5 LOC error \\ on predicted CLS} \\
+\hline
+VGG's \cite{Simonyan2015} & VGG-16 & 1-crop & 33.1 \cite{Simonyan2015} & & \\
+RPN & ResNet-101 & 1-crop & 13.3 & & \\
+RPN & ResNet-101 & dense & 11.7 & & \\
+\hline
+RPN & ResNet-101 & dense & & ResNet-101 & 14.4 \\
+RPN+RCNN & ResNet-101 & dense & & ResNet-101 & \textbf{10.6} \\
+RPN+RCNN & ensemble & dense & & ensemble & \textbf{8.9} \\
+\hline
+\end{tabular}
+}
+\end{center}
+\vspace{-.5em}
+\caption{Localization error (\%) on the ImageNet validation. In the column of ``LOC error on GT class'' (\cite{Simonyan2015}), the ground truth class is used.
+In the ``testing'' column, ``1-crop'' denotes testing on a center crop of 224$\times$224 pixels, ``dense'' denotes dense (fully convolutional) and multi-scale testing.
+}
+\vspace{-.5em}
+\label{tab:localization}
+\end{table}
+
+The ImageNet Localization (LOC) task \cite{Russakovsky2014} requires to classify and localize the objects.
+Following \cite{Sermanet2014,Simonyan2015}, we assume that the image-level classifiers are first adopted for predicting the class labels of an image, and the localization algorithm only accounts for predicting bounding boxes based on the predicted classes. We adopt the ``per-class regression'' (PCR) strategy \cite{Sermanet2014,Simonyan2015}, learning a bounding box regressor for each class. We pre-train the networks for ImageNet classification and then fine-tune them for localization.
+We train networks on the provided 1000-class ImageNet training set.
+
+Our localization algorithm is based on the RPN framework of \cite{Ren2015} with a few modifications.
+Unlike the way in \cite{Ren2015} that is category-agnostic, our RPN for localization is designed in a \emph{per-class} form. This RPN ends with two sibling 1$\times$1 convolutional layers for binary classification (\emph{cls}) and box regression (\emph{reg}), as in \cite{Ren2015}. The \emph{cls} and \emph{reg} layers are both in a \emph{per-class} from, in contrast to \cite{Ren2015}. Specifically, the \emph{cls} layer has a 1000-d output, and each dimension is \emph{binary logistic regression} for predicting being or not being an object class; the \emph{reg} layer has a 1000$\times$4-d output consisting of box regressors for 1000 classes.
+As in \cite{Ren2015}, our bounding box regression is with reference to multiple translation-invariant ``anchor'' boxes at each position.
+
+As in our ImageNet classification training (Sec.~\ref{sec:impl}), we randomly sample 224$\times$224 crops for data augmentation. We use a mini-batch size of 256 images for fine-tuning.
+To avoid negative samples being dominate, 8 anchors are randomly sampled for each image, where the sampled positive and negative anchors have a ratio of 1:1 \cite{Ren2015}. For testing, the network is applied on the image fully-convolutionally.
+
+\renewcommand\arraystretch{1.05}
+\setlength{\tabcolsep}{10pt}
+\begin{table}[t]
+\begin{center}
+\small
+\begin{tabular}{l|c|c}
+\hline
+  \multicolumn{1}{c|}{\multirow{2}{*}{method}}  & \multicolumn{2}{c}{top-5 localization err} \\\cline{2-3}
+  & val & test \\
+\hline
+OverFeat \cite{Sermanet2014} (ILSVRC'13) & 30.0 & 29.9 \\
+GoogLeNet \cite{Szegedy2015} (ILSVRC'14) & - & 26.7 \\
+VGG \cite{Simonyan2015} (ILSVRC'14) & 26.9 & 25.3 \\
+\hline
+ours (ILSVRC'15) & \textbf{8.9} & \textbf{9.0} \\
+\hline
+\end{tabular}
+\end{center}
+\vspace{-.5em}
+\caption{Comparisons of localization error (\%) on the ImageNet dataset with state-of-the-art methods.
+}
+\vspace{-.5em}
+\label{tab:localization_all}
+\end{table}
+
+Table~\ref{tab:localization} compares the localization results. Following \cite{Simonyan2015}, we first perform ``oracle'' testing using the ground truth class as the classification prediction. VGG's paper \cite{Simonyan2015} reports a center-crop error of 33.1\% (Table~\ref{tab:localization}) using ground truth classes. Under the same setting, our RPN method using ResNet-101 net significantly reduces the center-crop error to 13.3\%. This comparison demonstrates the excellent performance of our framework.
+With dense (fully convolutional) and multi-scale testing, our ResNet-101 has an error of 11.7\% using ground truth classes. Using ResNet-101 for predicting classes (4.6\% top-5 classification error, Table~\ref{tab:single}), the top-5 localization error is 14.4\%.
+
+The above results are only based on the \emph{proposal network} (RPN) in Faster R-CNN \cite{Ren2015}. One may use the \emph{detection network} (Fast R-CNN \cite{Girshick2015}) in Faster R-CNN to improve the results. But we notice that on this dataset, one image usually contains a single dominate object, and the proposal regions highly overlap with each other and thus have very similar RoI-pooled features. As a result, the image-centric training of Fast R-CNN \cite{Girshick2015} generates samples of small variations, which may not be desired for stochastic training. Motivated by this, in our current experiment we use the original R-CNN \cite{Girshick2014} that is RoI-centric, in place of Fast R-CNN.
+
+Our R-CNN implementation is as follows. We apply the per-class RPN trained as above on the training images to predict bounding boxes for the ground truth class. These predicted boxes play a role of class-dependent proposals.
+For each training image, the highest scored 200 proposals are extracted as training samples to train an R-CNN classifier. The image region is cropped from a proposal, warped to 224$\times$224 pixels, and fed into the classification network as in R-CNN \cite{Girshick2014}. The outputs of this network consist of two sibling fc layers for \emph{cls} and \emph{reg}, also in a per-class form.
+This R-CNN network is fine-tuned on the training set using a mini-batch size of 256 in the RoI-centric fashion. For testing, the RPN generates the highest scored 200 proposals for each predicted class, and the R-CNN network is used to update these proposals' scores and box positions.
+
+This method reduces the top-5 localization error to 10.6\% (Table~\ref{tab:localization}). This is our single-model result on the validation set. Using an ensemble of networks for both classification and localization, we achieve a top-5 localization error of 9.0\% on the test set. This number significantly outperforms the ILSVRC 14 results (Table~\ref{tab:localization_all}), showing a 64\% relative reduction of error. \emph{This result won the 1st place in the ImageNet localization task in ILSVRC 2015.}
+
+
+\end{document}