How to push on hub a quantized model
I quantized my model using torch but when I try to push model_quantized_qint8 to the hub it don’t workfrom torch.quantization import quantize_dynamicmodel_ckpt = “Sohaibsoussi/distilbert-base-uncased-distilled-finetuned-clinc”tokenizer = AutoTokenizer.from_pretrained(model_ckpt)model = AutoModelForSequenceClassification.from_pretrained(model_ckpt).to(“cpu”)Dynamic Quantization for INT8model_quantized_qint8 = quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8){“name”: “TypeError”,“message”: “_batch_encode_plus() got an unexpected keyword argument ‘dtype’”,“stack”: "---------------------------------------------------------------------------TypeError                                 Traceback (most recent call last)Cell In[169], line 52 optim_type = "Distillation + Static Quantization"4 pb = PerformanceBenchmark(pipe, clinc["test"], optim_type=optim_type)----> 5 perf_metrics.update(pb.run_benchmark())Cell In[3], line 20, in PerformanceBenchmark.run_benchmark(self)18 metrics = {}19 metrics[self.optim_type] = self.compute_size()—> 20 metrics[self.optim_type].update(self.time_pipeline())21 metrics[self.optim_type].update(self.compute_accuracy())22 return metricsCell In[17], line 8, in time_pipeline(self, query)6 # Warmup7 for _ in range(10):----> 8   _ = self.pipeline(query)9 # Timed Run10 for _ in range(100):File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/pipelines/text_classification.py:155, in TextClassificationPipeline.call(self, *args, **kwargs)121 defcall(self, *args, **kwargs):122     """123     Classify the text(s) given as inputs.124(…)153         Iftop_kis used, one such dictionary is returned per label.154     """→ 155     result = super().call(*args, **kwargs)156     # TODO try and retrieve it in a nicer way from _sanitize_parameters.157     _legacy = "top_k" not in kwargsFile ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/pipelines/base.py:1196, in Pipeline.call(self, inputs, num_workers, batch_size, *args, **kwargs)1188     return next(1189         iter(1190             self.get_iterator((…)1193         )1194     )1195 else:→ 1196     return self.run_single(inputs, preprocess_params, forward_params, postprocess_params)File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/pipelines/base.py:1202, in Pipeline.run_single(self, inputs, preprocess_params, forward_params, postprocess_params)1201 def run_single(self, inputs, preprocess_params, forward_params, postprocess_params):→ 1202     model_inputs = self.preprocess(inputs, **preprocess_params)1203     model_outputs = self.forward(model_inputs, **forward_params)1204     outputs = self.postprocess(model_outputs, **postprocess_params)File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/pipelines/text_classification.py:179, in TextClassificationPipeline.preprocess(self, inputs, **tokenizer_kwargs)173 elif isinstance(inputs, list):174     # This is likely an invalid usage of the pipeline attempting to pass text pairs.175     raise ValueError(176         "The pipeline received invalid inputs, if you are trying to send text pairs, you can try to send a"177         ’ dictionary{\"text\": \"My text\", \"text_pair\": \"My pair\"}in order to send a text pair.’178     )→ 179 return self.tokenizer(inputs, return_tensors=return_tensors, **tokenizer_kwargs)File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2829, in PreTrainedTokenizerBase.call(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)2827     if not self._in_target_context_manager:2828         self._switch_to_input_mode()→ 2829     encodings = self._call_one(text=text, text_pair=text_pair, **all_kwargs)2830 if text_target is not None:2831     self._switch_to_target_mode()File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:2935, in PreTrainedTokenizerBase._call_one(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)2915     return self.batch_encode_plus(2916         batch_text_or_text_pairs=batch_text_or_text_pairs,2917         add_special_tokens=add_special_tokens,(…)2932         **kwargs,2933     )2934 else:→ 2935     return self.encode_plus(2936         text=text,2937         text_pair=text_pair,2938         add_special_tokens=add_special_tokens,2939         padding=padding,2940         truncation=truncation,2941         max_length=max_length,2942         stride=stride,2943         is_split_into_words=is_split_into_words,2944         pad_to_multiple_of=pad_to_multiple_of,2945         return_tensors=return_tensors,2946         return_token_type_ids=return_token_type_ids,2947         return_attention_mask=return_attention_mask,2948         return_overflowing_tokens=return_overflowing_tokens,2949         return_special_tokens_mask=return_special_tokens_mask,2950         return_offsets_mapping=return_offsets_mapping,2951         return_length=return_length,2952         verbose=verbose,2953         **kwargs,2954     )File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:3008, in PreTrainedTokenizerBase.encode_plus(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)2998 # Backward compatibility for ‘truncation_strategy’, ‘pad_to_max_length’2999 padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(3000     padding=padding,3001     truncation=truncation,(…)3005     **kwargs,3006 )→ 3008 return self._encode_plus(3009     text=text,3010     text_pair=text_pair,3011     add_special_tokens=add_special_tokens,3012     padding_strategy=padding_strategy,3013     truncation_strategy=truncation_strategy,3014     max_length=max_length,3015     stride=stride,3016     is_split_into_words=is_split_into_words,3017     pad_to_multiple_of=pad_to_multiple_of,3018     return_tensors=return_tensors,3019     return_token_type_ids=return_token_type_ids,3020     return_attention_mask=return_attention_mask,3021     return_overflowing_tokens=return_overflowing_tokens,3022     return_special_tokens_mask=return_special_tokens_mask,3023     return_offsets_mapping=return_offsets_mapping,3024     return_length=return_length,3025     verbose=verbose,3026     **kwargs,3027 )File ~/miniconda3/envs/langchain/lib/python3.8/site-packages/transformers/tokenization_utils_fast.py:576, in PreTrainedTokenizerFast._encode_plus(self, text, text_pair, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)554 def _encode_plus(555     self,556     text: Union[TextInput, PreTokenizedInput],(…)573     **kwargs,574 ) → BatchEncoding:575     batched_input = [(text, text_pair)] if text_pair else [text]→ 576     batched_output = self._batch_encode_plus(577         batched_input,578         is_split_into_words=is_split_into_words,579         add_special_tokens=add_special_tokens,580         padding_strategy=padding_strategy,581         truncation_strategy=truncation_strategy,582         max_length=max_length,583         stride=stride,584         pad_to_multiple_of=pad_to_multiple_of,585         return_tensors=return_tensors,586         return_token_type_ids=return_token_type_ids,587         return_attention_mask=return_attention_mask,588         return_overflowing_tokens=return_overflowing_tokens,589         return_special_tokens_mask=return_special_tokens_mask,590         return_offsets_mapping=return_offsets_mapping,591         return_length=return_length,592         verbose=verbose,593         **kwargs,594     )596     # Return tensor is None, then we can remove the leading batch axis597     # Overflowing tokens are returned as a batch of output so we keep them in this case598     if return_tensors is None and not return_overflowing_tokens:TypeError: _batch_encode_plus() got an unexpected keyword argument ‘dtype’"}