(* This file is part of Ubase *)

let non_ascii_white_space = List.filter (fun i -> i > 127) Ubase_data.white_space

(* This list is manually created and overrides the automatically generated list
   [latin_uchar_to_base_alist]. Please add your own. (Only non-ascii codes) *)
let misc_to_ascii_alist = [
  (* Letters *)
  0x00aa, "a"; (* "ª" = FEMININE ORDINAL INDICATOR *)
  0x00ba, "o"; (* "º" = MASCULINE ORDINAL INDICATOR *)
  0x00df, "ss";  (* "ß" = LATIN SMALL LETTER SHARP S *)

  (* Ponctuation & guillemets *)
  0x2010, "-"; (* HYPHEN *)
  0x2013, "-"; (* EN DASH *)
  0x2014, "-"; (* EM DASH *)
  0x2018, "'"; (* LEFT SINGLE QUOTATION MARK *)
  0x2019, "'"; (* RIGHT SINGLE QUOTATION MARK *)
  0x201c, "\""; (* LEFT DOUBLE QUOTATION MARK *)
  0x201d, "\""; (* RIGHT DOUBLE QUOTATION MARK *)
  0x2026, "..."; (* HORIZONTAL ELLIPSIS *)
  0x2212, "-"; (* MINUS SIGN *)
]
  |> List.append (List.map (fun i -> (i, " ")) non_ascii_white_space)


(* From
   http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT *)
let cp1252_to_utf8_array = [|
(* 0x80 *) 0x20AC; (* EURO SIGN *)
(* 0x81 *) 0; (* UNDEFINED *)
(* 0x82 *) 0x201A; (* SINGLE LOW-9 QUOTATION MARK *)
(* 0x83 *) 0x0192; (* LATIN SMALL LETTER F WITH HOOK *)
(* 0x84 *) 0x201E; (* DOUBLE LOW-9 QUOTATION MARK *)
(* 0x85 *) 0x2026; (* HORIZONTAL ELLIPSIS *)
(* 0x86 *) 0x2020; (* DAGGER *)
(* 0x87 *) 0x2021; (* DOUBLE DAGGER *)
(* 0x88 *) 0x02C6; (* MODIFIER LETTER CIRCUMFLEX ACCENT *)
(* 0x89 *) 0x2030; (* PER MILLE SIGN *)
(* 0x8A *) 0x0160; (* LATIN CAPITAL LETTER S WITH CARON *)
(* 0x8B *) 0x2039; (* SINGLE LEFT-POINTING ANGLE QUOTATION MARK *)
(* 0x8C *) 0x0152; (* LATIN CAPITAL LIGATURE OE *)
(* 0x8D *) 0; (* UNDEFINED *)
(* 0x8E *) 0x017D; (* LATIN CAPITAL LETTER Z WITH CARON *)
(* 0x8F *) 0; (* UNDEFINED *)
(* 0x90 *) 0; (* UNDEFINED *)
(* 0x91 *) 0x2018; (* LEFT SINGLE QUOTATION MARK *)
(* 0x92 *) 0x2019; (* RIGHT SINGLE QUOTATION MARK *)
(* 0x93 *) 0x201C; (* LEFT DOUBLE QUOTATION MARK *)
(* 0x94 *) 0x201D; (* RIGHT DOUBLE QUOTATION MARK *)
(* 0x95 *) 0x2022; (* BULLET *)
(* 0x96 *) 0x2013; (* EN DASH *)
(* 0x97 *) 0x2014; (* EM DASH *)
(* 0x98 *) 0x02DC; (* SMALL TILDE *)
(* 0x99 *) 0x2122; (* TRADE MARK SIGN *)
(* 0x9A *) 0x0161; (* LATIN SMALL LETTER S WITH CARON *)
(* 0x9B *) 0x203A; (* SINGLE RIGHT-POINTING ANGLE QUOTATION MARK *)
(* 0x9C *) 0x0153; (* LATIN SMALL LIGATURE OE *)
(* 0x9D *) 0; (* UNDEFINED *)
(* 0x9E *) 0x017E; (* LATIN SMALL LETTER Z WITH CARON *)
(* 0x9F *) 0x0178; (* LATIN CAPITAL LETTER Y WITH DIAERESIS *)
(* 0xA0 *) 0x00A0; (* NO-BREAK SPACE *)
(* 0xA1 *) 0x00A1; (* INVERTED EXCLAMATION MARK *)
(* 0xA2 *) 0x00A2; (* CENT SIGN *)
(* 0xA3 *) 0x00A3; (* POUND SIGN *)
(* 0xA4 *) 0x00A4; (* CURRENCY SIGN *)
(* 0xA5 *) 0x00A5; (* YEN SIGN *)
(* 0xA6 *) 0x00A6; (* BROKEN BAR *)
(* 0xA7 *) 0x00A7; (* SECTION SIGN *)
(* 0xA8 *) 0x00A8; (* DIAERESIS *)
(* 0xA9 *) 0x00A9; (* COPYRIGHT SIGN *)
(* 0xAA *) 0x00AA; (* FEMININE ORDINAL INDICATOR *)
(* 0xAB *) 0x00AB; (* LEFT-POINTING DOUBLE ANGLE QUOTATION MARK *)
(* 0xAC *) 0x00AC; (* NOT SIGN *)
(* 0xAD *) 0x00AD; (* SOFT HYPHEN *)
(* 0xAE *) 0x00AE; (* REGISTERED SIGN *)
(* 0xAF *) 0x00AF; (* MACRON *)
(* 0xB0 *) 0x00B0; (* DEGREE SIGN *)
(* 0xB1 *) 0x00B1; (* PLUS-MINUS SIGN *)
(* 0xB2 *) 0x00B2; (* SUPERSCRIPT TWO *)
(* 0xB3 *) 0x00B3; (* SUPERSCRIPT THREE *)
(* 0xB4 *) 0x00B4; (* ACUTE ACCENT *)
(* 0xB5 *) 0x00B5; (* MICRO SIGN *)
(* 0xB6 *) 0x00B6; (* PILCROW SIGN *)
(* 0xB7 *) 0x00B7; (* MIDDLE DOT *)
(* 0xB8 *) 0x00B8; (* CEDILLA *)
(* 0xB9 *) 0x00B9; (* SUPERSCRIPT ONE *)
(* 0xBA *) 0x00BA; (* MASCULINE ORDINAL INDICATOR *)
(* 0xBB *) 0x00BB; (* RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK *)
(* 0xBC *) 0x00BC; (* VULGAR FRACTION ONE QUARTER *)
(* 0xBD *) 0x00BD; (* VULGAR FRACTION ONE HALF *)
(* 0xBE *) 0x00BE; (* VULGAR FRACTION THREE QUARTERS *)
(* 0xBF *) 0x00BF; (* INVERTED QUESTION MARK *)
(* 0xC0 *) 0x00C0; (* LATIN CAPITAL LETTER A WITH GRAVE *)
(* 0xC1 *) 0x00C1; (* LATIN CAPITAL LETTER A WITH ACUTE *)
(* 0xC2 *) 0x00C2; (* LATIN CAPITAL LETTER A WITH CIRCUMFLEX *)
(* 0xC3 *) 0x00C3; (* LATIN CAPITAL LETTER A WITH TILDE *)
(* 0xC4 *) 0x00C4; (* LATIN CAPITAL LETTER A WITH DIAERESIS *)
(* 0xC5 *) 0x00C5; (* LATIN CAPITAL LETTER A WITH RING ABOVE *)
(* 0xC6 *) 0x00C6; (* LATIN CAPITAL LETTER AE *)
(* 0xC7 *) 0x00C7; (* LATIN CAPITAL LETTER C WITH CEDILLA *)
(* 0xC8 *) 0x00C8; (* LATIN CAPITAL LETTER E WITH GRAVE *)
(* 0xC9 *) 0x00C9; (* LATIN CAPITAL LETTER E WITH ACUTE *)
(* 0xCA *) 0x00CA; (* LATIN CAPITAL LETTER E WITH CIRCUMFLEX *)
(* 0xCB *) 0x00CB; (* LATIN CAPITAL LETTER E WITH DIAERESIS *)
(* 0xCC *) 0x00CC; (* LATIN CAPITAL LETTER I WITH GRAVE *)
(* 0xCD *) 0x00CD; (* LATIN CAPITAL LETTER I WITH ACUTE *)
(* 0xCE *) 0x00CE; (* LATIN CAPITAL LETTER I WITH CIRCUMFLEX *)
(* 0xCF *) 0x00CF; (* LATIN CAPITAL LETTER I WITH DIAERESIS *)
(* 0xD0 *) 0x00D0; (* LATIN CAPITAL LETTER ETH *)
(* 0xD1 *) 0x00D1; (* LATIN CAPITAL LETTER N WITH TILDE *)
(* 0xD2 *) 0x00D2; (* LATIN CAPITAL LETTER O WITH GRAVE *)
(* 0xD3 *) 0x00D3; (* LATIN CAPITAL LETTER O WITH ACUTE *)
(* 0xD4 *) 0x00D4; (* LATIN CAPITAL LETTER O WITH CIRCUMFLEX *)
(* 0xD5 *) 0x00D5; (* LATIN CAPITAL LETTER O WITH TILDE *)
(* 0xD6 *) 0x00D6; (* LATIN CAPITAL LETTER O WITH DIAERESIS *)
(* 0xD7 *) 0x00D7; (* MULTIPLICATION SIGN *)
(* 0xD8 *) 0x00D8; (* LATIN CAPITAL LETTER O WITH STROKE *)
(* 0xD9 *) 0x00D9; (* LATIN CAPITAL LETTER U WITH GRAVE *)
(* 0xDA *) 0x00DA; (* LATIN CAPITAL LETTER U WITH ACUTE *)
(* 0xDB *) 0x00DB; (* LATIN CAPITAL LETTER U WITH CIRCUMFLEX *)
(* 0xDC *) 0x00DC; (* LATIN CAPITAL LETTER U WITH DIAERESIS *)
(* 0xDD *) 0x00DD; (* LATIN CAPITAL LETTER Y WITH ACUTE *)
(* 0xDE *) 0x00DE; (* LATIN CAPITAL LETTER THORN *)
(* 0xDF *) 0x00DF; (* LATIN SMALL LETTER SHARP S *)
(* 0xE0 *) 0x00E0; (* LATIN SMALL LETTER A WITH GRAVE *)
(* 0xE1 *) 0x00E1; (* LATIN SMALL LETTER A WITH ACUTE *)
(* 0xE2 *) 0x00E2; (* LATIN SMALL LETTER A WITH CIRCUMFLEX *)
(* 0xE3 *) 0x00E3; (* LATIN SMALL LETTER A WITH TILDE *)
(* 0xE4 *) 0x00E4; (* LATIN SMALL LETTER A WITH DIAERESIS *)
(* 0xE5 *) 0x00E5; (* LATIN SMALL LETTER A WITH RING ABOVE *)
(* 0xE6 *) 0x00E6; (* LATIN SMALL LETTER AE *)
(* 0xE7 *) 0x00E7; (* LATIN SMALL LETTER C WITH CEDILLA *)
(* 0xE8 *) 0x00E8; (* LATIN SMALL LETTER E WITH GRAVE *)
(* 0xE9 *) 0x00E9; (* LATIN SMALL LETTER E WITH ACUTE *)
(* 0xEA *) 0x00EA; (* LATIN SMALL LETTER E WITH CIRCUMFLEX *)
(* 0xEB *) 0x00EB; (* LATIN SMALL LETTER E WITH DIAERESIS *)
(* 0xEC *) 0x00EC; (* LATIN SMALL LETTER I WITH GRAVE *)
(* 0xED *) 0x00ED; (* LATIN SMALL LETTER I WITH ACUTE *)
(* 0xEE *) 0x00EE; (* LATIN SMALL LETTER I WITH CIRCUMFLEX *)
(* 0xEF *) 0x00EF; (* LATIN SMALL LETTER I WITH DIAERESIS *)
(* 0xF0 *) 0x00F0; (* LATIN SMALL LETTER ETH *)
(* 0xF1 *) 0x00F1; (* LATIN SMALL LETTER N WITH TILDE *)
(* 0xF2 *) 0x00F2; (* LATIN SMALL LETTER O WITH GRAVE *)
(* 0xF3 *) 0x00F3; (* LATIN SMALL LETTER O WITH ACUTE *)
(* 0xF4 *) 0x00F4; (* LATIN SMALL LETTER O WITH CIRCUMFLEX *)
(* 0xF5 *) 0x00F5; (* LATIN SMALL LETTER O WITH TILDE *)
(* 0xF6 *) 0x00F6; (* LATIN SMALL LETTER O WITH DIAERESIS *)
(* 0xF7 *) 0x00F7; (* DIVISION SIGN *)
(* 0xF8 *) 0x00F8; (* LATIN SMALL LETTER O WITH STROKE *)
(* 0xF9 *) 0x00F9; (* LATIN SMALL LETTER U WITH GRAVE *)
(* 0xFA *) 0x00FA; (* LATIN SMALL LETTER U WITH ACUTE *)
(* 0xFB *) 0x00FB; (* LATIN SMALL LETTER U WITH CIRCUMFLEX *)
(* 0xFC *) 0x00FC; (* LATIN SMALL LETTER U WITH DIAERESIS *)
(* 0xFD *) 0x00FD; (* LATIN SMALL LETTER Y WITH ACUTE *)
(* 0xFE *) 0x00FE; (* LATIN SMALL LETTER THORN *)
(* 0xFF *) 0x00FF; (* LATIN SMALL LETTER Y WITH DIAERESIS *)
  |]
