text: # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import ssl
  import re
  import urllib.request
  import json
  import collections
  import sys
  import getopt
  import external_error_pb2
  from html.parser import HTMLParser
  
  
  def parsing(externalErrorDesc):
      #*********************************************************************************************#
      #*********************************** CUDA Error Message **************************************#
      print("start crawling errorMessage for nvidia CUDA API--->")
      url = 'https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CUDA
  
      ssl._create_default_https_context = ssl._create_unverified_context
      html = urllib.request.urlopen(url).read().decode('utf-8')
      res_div = r'<div class="section">.*?<p>CUDA error types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
      for error in m_dt:
          res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
          m_type = re.findall(res_type, error[0], re.S | re.M)[0]
          m_message = error[1]
          m_message = m_message.replace('\n', '')
          res_a = r'(<a class=.*?</a>)'
          res_shape = r'<a class=.*?>(.*?)</a>'
          list_a = re.findall(res_a, m_message, re.S | re.M)
          list_shape = re.findall(res_shape, m_message, re.S | re.M)
          assert len(list_a) == len(list_shape)
          for idx in range(len(list_a)):
              m_message = m_message.replace(list_a[idx], list_shape[idx])
  
          m_message = m_message.replace(
              '<h6 class=\"deprecated_header\">Deprecated</h6>', '')
  
          res_span = r'(<span class=.*?</span>)'
          res_span_detail = r'<span class=.*?>(.*?)</span>'
          list_span = re.findall(res_span, m_message, re.S | re.M)
          list_span_detail = re.findall(res_span_detail, m_message, re.S | re.M)
          assert len(list_span) == len(list_span_detail)
          for idx in range(len(list_span)):
              m_message = m_message.replace(list_span[idx], list_span_detail[idx])
  
          res_p = r'(<p>.*?</p>)'
          res_p_detail = r'<p>(.*?)</p>'
          list_p = re.findall(res_p, m_message, re.S | re.M)
          list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
          assert len(list_p) == len(list_p_detail)
          for idx in range(len(list_p)):
              m_message = m_message.replace(list_p[idx], list_p_detail[idx])
  
          m_message = m_message.replace('  ', '')
          _Messages = allMessageDesc.messages.add()
          try:
              _Messages.code = int(m_type[1])
          except ValueError:
              if re.match('0x', m_type[1]):
                  _Messages.code = int(m_type[1], 16)
              else:
                  raise ValueError
          _Messages.message = "'%s'. %s" % (m_type[0], m_message)
      print("End crawling errorMessage for nvidia CUDA API!\n")
  
      #***********************************************************************************************#
      #*********************************** CURAND Error Message **************************************#
      print("start crawling errorMessage for nvidia CURAND API--->")
      url = 'https://docs.nvidia.com/cuda/curand/group__HOST.html#group__HOST_1gb94a31d5c165858c96b6c18b70644437'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CURAND
  
      html = urllib.request.urlopen(url).read().decode('utf-8')
  
      res_div = r'<div class="section">.*?<p>CURAND function call status types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
      for error in m_dt:
          res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
          m_type = re.findall(res_type, error[0], re.S | re.M)[0]
          m_message = error[1]
  
          _Messages = allMessageDesc.messages.add()
          try:
              _Messages.code = int(m_type[1])
          except ValueError:
              if re.match('0x', m_type[1]):
                  _Messages.code = int(m_type[1], 16)
              else:
                  raise ValueError
          _Messages.message = "'%s'. %s" % (m_type[0], m_message)
      print("End crawling errorMessage for nvidia CURAND API!\n")
  
      #**************************************************************************************************#
      #*********************************** CUDNN Error Message ******************************************#
      cudnnStatus_t = {
          "CUDNN_STATUS_SUCCESS": 0,
          "CUDNN_STATUS_NOT_INITIALIZED": 1,
          "CUDNN_STATUS_ALLOC_FAILED": 2,
          "CUDNN_STATUS_BAD_PARAM": 3,
          "CUDNN_STATUS_INTERNAL_ERROR": 4,
          "CUDNN_STATUS_INVALID_VALUE": 5,
          "CUDNN_STATUS_ARCH_MISMATCH": 6,
          "CUDNN_STATUS_MAPPING_ERROR": 7,
          "CUDNN_STATUS_EXECUTION_FAILED": 8,
          "CUDNN_STATUS_NOT_SUPPORTED": 9,
          "CUDNN_STATUS_LICENSE_ERROR": 10,
          "CUDNN_STATUS_RUNTIME_PREREQUISITE_MISSING": 11,
          "CUDNN_STATUS_RUNTIME_IN_PROGRESS": 12,
          "CUDNN_STATUS_RUNTIME_FP_OVERFLOW": 13,
      }
  
      print("start crawling errorMessage for nvidia CUDNN API--->")
      url = 'https://docs.nvidia.com/deeplearning/cudnn/api/index.html#cudnnStatus_t'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CUDNN
  
      html = urllib.request.urlopen(url).read().decode('utf-8')
      f = open('1.txt', 'w')
      f.write(html)
  
      res_div = r'<div class="section" id="cudnnStatus_t__section_lmp_dgr_2jb"><a name="cudnnStatus_t__section_lmp_dgr_2jb" shape="rect">(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<dt class="dt dlterm"><samp class="ph codeph">(.*?)</samp></dt>.*?<dd class="dd">(.*?)</dd>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
      for error in m_dt:
          m_message = error[1]
  
          res_class = r'<p class="p">.*?</p>'
          res_class_detail = r'<p class="p">(.*?)</p>'
          list_class = re.findall(res_class, m_message, re.S | re.M)
          list_class_detail = re.findall(res_class_detail, m_message, re.S | re.M)
          assert len(list_class) == len(list_class_detail)
          for idx in range(len(list_class)):
              m_message = m_message.replace(list_class[idx],
                                            list_class_detail[idx])
  
          res_a = r'(<a class="xref".*?</a>)'
          res_shape = r'<a class="xref".*?>(.*?)</a>'
          list_a = re.findall(res_a, m_message, re.S | re.M)
          list_shape = re.findall(res_shape, m_message, re.S | re.M)
          assert len(list_a) == len(list_shape)
          for idx in range(len(list_a)):
              m_message = m_message.replace(list_a[idx], list_shape[idx])
  
          res_span = r'(<span class="ph">.*?</span>)'
          res_span_detail = r'<span class="ph">(.*?)</span>'
          list_span = re.findall(res_span, m_message, re.S | re.M)
          list_span_detail = re.findall(res_span_detail, m_message, re.S | re.M)
          assert len(list_span) == len(list_span_detail)
          for idx in range(len(list_span)):
              m_message = m_message.replace(list_span[idx], list_span_detail[idx])
  
          res_samp = r'(<samp class="ph codeph">.*?</samp>)'
          res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
          list_samp = re.findall(res_samp, m_message, re.S | re.M)
          list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
          assert len(list_samp) == len(list_samp_detail)
          for idx in range(len(list_samp)):
              m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
  
          m_message = re.sub(r'\n +', ' ', m_message)
  
          _Messages = allMessageDesc.messages.add()
          _Messages.code = int(cudnnStatus_t[error[0]])
          _Messages.message = "'%s'. %s" % (error[0], m_message)
      print("End crawling errorMessage for nvidia CUDNN API!\n")
  
      #*************************************************************************************************#
      #*********************************** CUBLAS Error Message ****************************************#
      cublasStatus_t = {
          "CUBLAS_STATUS_SUCCESS": 0,
          "CUBLAS_STATUS_NOT_INITIALIZED": 1,
          "CUBLAS_STATUS_ALLOC_FAILED": 3,
          "CUBLAS_STATUS_INVALID_VALUE": 7,
          "CUBLAS_STATUS_ARCH_MISMATCH": 8,
          "CUBLAS_STATUS_MAPPING_ERROR": 11,
          "CUBLAS_STATUS_EXECUTION_FAILED": 13,
          "CUBLAS_STATUS_INTERNAL_ERROR": 14,
          "CUBLAS_STATUS_NOT_SUPPORTED": 15,
          "CUBLAS_STATUS_LICENSE_ERROR": 16
      }
  
      print("start crawling errorMessage for nvidia CUBLAS API--->")
      url = 'https://docs.nvidia.com/cuda/cublas/index.html#cublasstatus_t'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CUBLAS
  
      html = urllib.request.urlopen(url).read().decode('utf-8')
  
      res_div = r'<p class="p">The type is used for function status returns. All cuBLAS library.*?<div class="tablenoborder">(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<p class="p"><samp class="ph codeph">(.*?)</samp></p>.*?colspan="1">(.*?)</td>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
  
      for error in m_dt:
          m_message = error[1]
          m_message = re.sub(r'\n +', ' ', m_message)
  
          res_p = r'<p class="p">.*?</p>'
          res_p_detail = r'<p class="p">(.*?)</p>'
          list_p = re.findall(res_p, m_message, re.S | re.M)
          list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
          assert len(list_p) == len(list_p_detail)
          for idx in range(len(list_p)):
              m_message = m_message.replace(list_p[idx], list_p_detail[idx])
  
          res_samp = r'<samp class="ph codeph">.*?</samp>'
          res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
          list_samp = re.findall(res_samp, m_message, re.S | re.M)
          list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
          assert len(list_samp) == len(list_samp_detail)
          for idx in range(len(list_samp)):
              m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
  
          _Messages = allMessageDesc.messages.add()
          _Messages.code = int(cublasStatus_t[error[0]])
          _Messages.message = "'%s'. %s" % (error[0], m_message)
      print("End crawling errorMessage for nvidia CUBLAS API!\n")
  
      #*************************************************************************************************#
      #*********************************** CUSOLVER Error Message **************************************#
      cusolverStatus_t = {
          "CUSOLVER_STATUS_SUCCESS": 0,
          "CUSOLVER_STATUS_NOT_INITIALIZED": 1,
          "CUSOLVER_STATUS_ALLOC_FAILED": 2,
          "CUSOLVER_STATUS_INVALID_VALUE": 3,
          "CUSOLVER_STATUS_ARCH_MISMATCH": 4,
          "CUSOLVER_STATUS_MAPPING_ERROR": 5,
          "CUSOLVER_STATUS_EXECUTION_FAILED": 6,
          "CUSOLVER_STATUS_INTERNAL_ERROR": 7,
          "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED": 8,
          "CUSOLVER_STATUS_NOT_SUPPORTED": 9,
          "CUSOLVER_STATUS_ZERO_PIVOT": 10,
          "CUSOLVER_STATUS_INVALID_LICENSE": 11,
          "CUSOLVER_STATUS_IRS_PARAMS_NOT_INITIALIZED": 12,
          "CUSOLVER_STATUS_IRS_PARAMS_INVALID": 13,
          "CUSOLVER_STATUS_IRS_INTERNAL_ERROR": 14,
          "CUSOLVER_STATUS_IRS_NOT_SUPPORTED": 15,
          "CUSOLVER_STATUS_IRS_OUT_OF_RANGE": 16,
          "CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES": 17,
          "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18
      }
      print("start crawling errorMessage for nvidia CUSOLVER API--->")
      url = 'https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverSPstatus'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CUSOLVER
  
      html = urllib.request.urlopen(url).read().decode('utf-8')
  
      res_div = r'This is a status type returned by the library functions and.*?<div class="tablenoborder">(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<samp class="ph codeph">(.*?)</samp></td>.*?colspan="1">(.*?)</td>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
  
      for error in m_dt:
          m_message = error[1]
          m_message = re.sub(r'\n +', '', m_message)
          m_message = re.sub(r'<p class="p"></p>', '', m_message)
  
          res_p = r'<p class="p">.*?</p>'
          res_p_detail = r'<p class="p">(.*?)</p>'
          list_p = re.findall(res_p, m_message, re.S | re.M)
          list_p_detail = re.findall(res_p_detail, m_message, re.S | re.M)
          assert len(list_p) == len(list_p_detail)
          for idx in range(len(list_p)):
              m_message = m_message.replace(list_p[idx], list_p_detail[idx])
  
          res_samp = r'<samp class="ph codeph">.*?</samp>'
          res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
          list_samp = re.findall(res_samp, m_message, re.S | re.M)
          list_samp_detail = re.findall(res_samp_detail, m_message, re.S | re.M)
          assert len(list_samp) == len(list_samp_detail)
          for idx in range(len(list_samp)):
              m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
  
          res_strong = r'<strong class="ph b">.*?</strong>'
          res_strong_detail = r'<strong class="ph b">(.*?)</strong>'
          list_strong = re.findall(res_strong, m_message, re.S | re.M)
          list_strong_detail = re.findall(res_strong_detail, m_message,
                                          re.S | re.M)
          assert len(list_strong) == len(list_strong_detail)
          for idx in range(len(list_strong)):
              m_message = m_message.replace(list_strong[idx],
                                            list_strong_detail[idx])
  
          _Messages = allMessageDesc.messages.add()
          _Messages.code = int(cusolverStatus_t[error[0]])
          _Messages.message = "'%s'. %s" % (error[0], m_message)
      print("End crawling errorMessage for nvidia CUSOLVER API!\n")
  
      #**********************************************************************************************#
      #*************************************** NCCL error *******************************************#
      print("start crawling errorMessage for nvidia NCCL API--->")
      url = 'https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclresult-t'
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.NCCL
      html = urllib.request.urlopen(url).read().decode('utf-8')
      res_div = r'<code class="descname">ncclResult_t</code>(.*?)</div>'
      m_div = re.findall(res_div, html, re.S | re.M)[0]
  
      res_dt = r'<code class="descname">(.*?)</code>.*?<span class="pre">(.*?)</span></code>\)(.*?)</p>\n</dd></dl>'
      m_dt = re.findall(res_dt, m_div, re.S | re.M)
      for error in m_dt:
          m_message = re.sub(r'\n', '', error[2])
          _Messages = allMessageDesc.messages.add()
          _Messages.code = int(error[1])
          _Messages.message = "'%s'. %s" % (error[0], m_message)
      print("End crawling errorMessage for nvidia NCCL API!\n")
  
      #*************************************************************************************************#
      #*********************************** CUFFT Error Message **************************************#
      print("start crawling errorMessage for nvidia CUFFT API--->")
      url = 'https://docs.nvidia.com/cuda/cufft/index.html#cufftresult'
  
      allMessageDesc = externalErrorDesc.errors.add()
      allMessageDesc.type = external_error_pb2.CUFFT
  
      html = urllib.request.urlopen(url).read().decode('utf-8')
  
      class CUFFTHTMLParser(HTMLParser):
          '''CUFFTHTML Parser
          '''
  
          def handle_data(self, data):
              if 'typedef enum cufftResult_t' in data:
                  for line in data.strip().splitlines()[1:-1]:
                      status, code, desc = re.split('=|//', line.strip())
                      _Messages = allMessageDesc.messages.add()
                      _Messages.code = int(code.strip(' ,'))
                      _Messages.message = "'%s'. %s" % (status.strip(),
                                                        desc.strip())
  
      CUFFTHTMLParser().feed(html)
  
  
  def main(argv):
      try:
          opts, _ = getopt.getopt(argv, "h", ["help"])
      except getopt.GetoptError:
          print('python spider.py')
          sys.exit(2)
      for opt, _ in opts:
          if opt in ("-h", "--help"):
              print('python spider.py')
              sys.exit(2)
      externalErrorDesc = external_error_pb2.ExternalErrorDesc()
      parsing(externalErrorDesc)
  
      serializedString = externalErrorDesc.SerializeToString()
      with open("externalErrorMsg.pb", "wb") as f:
          # save for externalErrorMsg.pb from Python-protobuf interface
          # load from C++-protobuf interface and get error message
          f.write(serializedString)
      print(
          "Generating data file [externalErrorMsg.pb] for external third_party API error has been done!"
      )
  
  
  if __name__ == "__main__":
      main(sys.argv[1:])
  