Update data_cleaning.py

data_cleaning.py

@@ -10,13 +10,19 @@ def clean_data(df):
     df['ToWhomTheyReplied'] = df['ToWhomTheyReplied'].fillna(df['Channel'])
     
     # Drop rows where 'Reply' column is missing
+    before = df.shape[0]
     df.dropna(subset=['Reply'], inplace=True)
+    after = df.shape[0]
     
     # Calculate comment and reply lengths
     df['comment_length'] = df['Comment'].str.len()
     df['reply_length'] = df['Reply'].str.len()
     
     # Remove duplicate rows
-    df = df.drop_duplicates()
+    num_duplicates = df.duplicated().sum()
+    df_deduplicated = df.drop_duplicates()
     
-    return df
+    # Print number of duplicates
+    print('Number of duplicate rows:', num_duplicates)
+    
+    return df_deduplicated