Subject: Re: SV: [xsl] output encoding problem (can't get UTF-8) From: Soren Kuula <dongfang@xxxxxxxxxxx> Date: Sun, 07 Aug 2005 00:18:52 +0200 |
That's what I'm thinking too, but the output has screwed up E D V in allHi,
places, not just links. Very weird. I can't think of why this would happen.
:/
Many people are not erally aware what this encoding thing is about... are you sure that the encoding you have DECLARED is the same as the one you have APPLIED throughout? The encoding you declare in xml is that in <?xml encoding="foobar"?>. The encoding you actually use for the document is the one that your editor, or the tool that generated it, saved it in. They must be the same.......
- Something writes the document, in utf-8. - Something reads it, thinking/guessing/defaulting to/being fooled into thinking that it's Latin-1 or some similar a-character-is-a-byte encoding. - The above is repeated zero or more times.
(the opposite phonomenon, characters disappering, is when programs think they're reading utf-8 but the data is not utf-8...)
I cooked up some Java once to demonstrate it: Utf-8 is digested as it were being one byte a char (as Latin-1). This is repeated, a big mess is made, and the opposite error is then applied twice to clean it up :)
C-x <return> C <now enter the name of the encoding that the xml header claims is used> <return> <C-x><C-f><enter filename>. That is: force emacs to use a specific encoding for the folling command, which is then 'read the file'. If now f x e d v are bad, then the encoding in the xml header is lying.....
One more thing: Can you use a mail client that writes Re: in the header, not SV: ? We get confused... (and mail ends up in the wrong folder)
private static int headmask(int headvalue) { if (headvalue >= 0xFC) return 0x1; if (headvalue >= 0xF8) return 0x3; if (headvalue >= 0xF0) return 0x7; if (headvalue >= 0xE0) return 0xF; if (headvalue >= 0xC0) return 0x1F; return 0x7F; }
public static int decode(int[] bytes, int startpos, int[] resulteater, int resultpos) { int numTrailers = taillength_by_headvalue(bytes[startpos]); int result = 0; int displacement = 0; for (int i=numTrailers; i>0; i--) { result+=(bytes[startpos + i] & 0x3F) << displacement; displacement += 6; } result += (bytes[startpos] & headmask(bytes[startpos])) << displacement; resulteater[resultpos] = result; return startpos + numTrailers + 1; }
public static int decodeAll(int[] bytes, int startpos, int stoppos, int[] result, int resultpos) { while(startpos < stoppos) { startpos=decode(bytes, startpos, result, resultpos++); } return resultpos; }
public static int decodeAll(int[] bytes, int[] result) { return decodeAll(bytes, 0, bytes.length, result, 0); }
private static int taillength_by_codepoint(int data) { if (data < 0x80) return 0; if (data < 0x800) return 1; if (data < 0x10000) return 2; if (data < 0x200000) return 3; if (data < 0x4000000) return 4; return 5; }
private static int headvalue_by_codepoint(int data) { if (data < 0x80) return 0; if (data < 0x800) return 0xC0; if (data < 0x10000) return 0xE0; if (data < 0x200000) return 0xF0; if (data < 0x4000000) return 0xF8; return 0xFC; }
public static int encode(int data, int[] result, int resultpos) { int length = taillength_by_codepoint(data); int temp = data; for (int i=length; i>0; i--) { result[resultpos + i] = (temp & 0x3F) | 0x80; temp >>>=6; } result[resultpos] = temp | headvalue_by_codepoint(data); return resultpos + length+1; }
public static int encode_many(int[] data, int startpos, int stoppos, int[] result, int resultpos) { for (int i=startpos; i<stoppos; i++) { resultpos=encode(data[i], result, resultpos); } return resultpos; }
public static int encode_all(int[] data, int[] result, int resultpos) { return encode_many(data, 0, data.length, result, resultpos); }
public static void dump(int[] data, int no) { for (int i=0; i<no; i++) { System.out.print(data[i]); // + "(" + (char)result[i] + ")"); if (i<no-1) System.out.print(", "); } System.out.print(" ("); for (int i=0; i<no; i++) { System.out.print((char)data[i]); } System.out.println(")"); }
public static void main (String[] args) { int[] singlycoded = new int[1000]; //int no = decodeAll(new int[]{0xC3, 0x83, 0xE8, 0x80, 0xA6}, result); int[] original = new int[]{'f','x','e','F','X','E'};
System.out.println("\nThis string (Unicode code points in paranthesis):"); dump(original, original.length);
int sth = encode_all(original, singlycoded, 0); System.out.println("\nis encoded into these UTF-8 bytes. If that is then improperly interpreted as Latin-1, it will be interpreted as the characters in the parenthesis"); dump(singlycoded, sth);
int[] doublycoded = new int[1000]; sth = encode_many(singlycoded, 0, sth, doublycoded, 0); System.out.println("\nThe bad interpretation encoded as Unicode. In the parenthesis are the results of repeating the above error of interpreting UTF-8 as Latin-1"); dump(doublycoded, sth);
int[] decoded = new int[1000]; sth = decodeAll(doublycoded, 0, sth, decoded, 0); System.out.println("\nDecoding the above yields:"); dump(decoded, sth);
int[] decoded_again = new int[1000]; sth = decodeAll(decoded, 0, sth, decoded_again, 0); System.out.println("\nDecoding again gives us:"); dump(decoded_again, sth); } }
Current Thread |
---|
|
<- Previous | Index | Next -> |
---|---|---|
RE: [xsl] output encoding problem (, Gabriel K | Thread | RE: [xsl] output encoding problem (, Gabriel K |
RE: [xsl] XSLT Editor, Colin Toal | Date | [xsl] Re: xml:space problem WAS : P, Kenneth Stephen |
Month |