GithubStar/fastjson
1
0
Fork 0
mirror of https://github.com/alibaba/fastjson.git synced 2026-10-22 01:06:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

bug fixed for stream reader api

Browse Source
This commit is contained in:
wenshao
2013-07-25 15:28:16 +08:00
parent 91f1244e6b
commit dad4e751ec
9 changed files with 232 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/META-INF/MANIFEST.MF
+1 -1
View File
@@ -1,3 +1,3 @@
Manifest-Version: 1.0
Manifest-Version: 1.1.35
Class-Path:
src/main/java/com/alibaba/fastjson/parser/JSONLexerBase.java
+46 -31
View File
@@ -396,6 +396,9 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
public final Number integerValue() throws NumberFormatException {
long result = 0;
boolean negative = false;
if (np == -1) {
np = 0;
}
int i = np, max = np + sp;
long limit;
long multmin;
@@ -636,7 +639,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
boolean hasSpecial = false;
char chLocal;
for (;;) {
chLocal = charAt(++bp);
chLocal = next();
if (chLocal == quote) {
break;
@@ -665,7 +668,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
arrayCopy(np + 1, sbuf, 0, sp);
}
chLocal = charAt(++bp);
chLocal = next();
switch (chLocal) {
case '0':
@@ -742,8 +745,8 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
putChar('\\');
break;
case 'x':
char x1 = ch = charAt(++bp);
char x2 = ch = charAt(++bp);
char x1 = ch = next();
char x2 = ch = next();
int x_val = digits[x1] * 16 + digits[x2];
char x_char = (char) x_val;
@@ -751,10 +754,10 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
putChar(x_char);
break;
case 'u':
char c1 = chLocal = charAt(++bp);
char c2 = chLocal = charAt(++bp);
char c3 = chLocal = charAt(++bp);
char c4 = chLocal = charAt(++bp);
char c1 = chLocal = next();
char c2 = chLocal = next();
char c3 = chLocal = next();
char c4 = chLocal = next();
int val = Integer.parseInt(new String(new char[] { c1, c2, c3, c4 }), 16);
hash = 31 * hash + val;
putChar((char) val);
@@ -781,14 +784,25 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
}
token = LITERAL_STRING;
this.next();
String value;
if (!hasSpecial) {
// return this.text.substring(np + 1, np + 1 + sp).intern();
return addSymbol(np + 1, sp, hash, symbolTable);
int offset;
if (np == -1) {
offset = 0;
} else {
offset = np + 1;
}
value = addSymbol(offset, sp, hash, symbolTable);
} else {
return symbolTable.addSymbol(sbuf, 0, sp, hash);
value = symbolTable.addSymbol(sbuf, 0, sp, hash);
}
sp = 0;
this.next();
return value;
}
public final void resetStringPosition() {
@@ -812,7 +826,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
sp = 1;
char chLocal;
for (;;) {
chLocal = charAt(++bp);
chLocal = next();
if (chLocal < identifierFlags.length) {
if (!identifierFlags[chLocal]) {
@@ -848,7 +862,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
hasSpecial = false;
char ch;
for (;;) {
ch = charAt(++bp);
ch = next();
if (ch == '\"') {
break;
@@ -877,7 +891,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
// System.arraycopy(buf, np + 1, sbuf, 0, sp);
}
ch = charAt(++bp);
ch = next();
switch (ch) {
case '0':
@@ -936,18 +950,18 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
putChar('\\');
break;
case 'x':
char x1 = ch = charAt(++bp);
char x2 = ch = charAt(++bp);
char x1 = ch = next();
char x2 = ch = next();
int x_val = digits[x1] * 16 + digits[x2];
char x_char = (char) x_val;
putChar(x_char);
break;
case 'u':
char u1 = ch = charAt(++bp);
char u2 = ch = charAt(++bp);
char u3 = ch = charAt(++bp);
char u4 = ch = charAt(++bp);
char u1 = ch = next();
char u2 = ch = next();
char u3 = ch = next();
char u4 = ch = next();
int val = Integer.parseInt(new String(new char[] { u1, u2, u3, u4 }), 16);
putChar((char) val);
break;
@@ -971,7 +985,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
}
token = JSONToken.LITERAL_STRING;
this.ch = charAt(++bp);
this.ch = next();
}
public Calendar getCalendar() {
@@ -1151,7 +1165,8 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
throw new JSONException("unclosed str");
}
String stringVal = subString(bp + fieldName.length + 1, endIndex - startIndex);
int startIndex2 = bp + fieldName.length + 1; // must re compute
String stringVal = subString(startIndex2, endIndex - startIndex2);
for (int i = bp + fieldName.length + 1; i < endIndex; ++i) {
if (charAt(i) == '\\') {
hasSpecial = true;
@@ -2538,7 +2553,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
hasSpecial = false;
char chLocal;
for (;;) {
chLocal = charAt(++bp);
chLocal = next();
if (chLocal == '\'') {
break;
@@ -2563,7 +2578,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
// System.arraycopy(buf, np + 1, sbuf, 0, sp);
}
chLocal = charAt(++bp);
chLocal = next();
switch (chLocal) {
case '0':
@@ -2622,18 +2637,18 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable {
putChar('\\');
break;
case 'x':
char x1 = chLocal = charAt(++bp);
char x2 = chLocal = charAt(++bp);
char x1 = chLocal = next();
char x2 = chLocal = next();
int x_val = digits[x1] * 16 + digits[x2];
char x_char = (char) x_val;
putChar(x_char);
break;
case 'u':
char c1 = chLocal = charAt(++bp);
char c2 = chLocal = charAt(++bp);
char c3 = chLocal = charAt(++bp);
char c4 = chLocal = charAt(++bp);
char c1 = chLocal = next();
char c2 = chLocal = next();
char c3 = chLocal = next();
char c4 = chLocal = next();
int val = Integer.parseInt(new String(new char[] { c1, c2, c3, c4 }), 16);
putChar((char) val);
break;
src/main/java/com/alibaba/fastjson/parser/JSONReaderScanner.java
+29 -12
View File
@@ -126,10 +126,11 @@ public final class JSONReaderScanner extends JSONLexerBase {
}
public final int indexOf(char ch, int startIndex) {
int offset = startIndex;
int offset = startIndex - bp;
for (;; ++offset) {
if (ch == charAt(offset)) {
return offset;
final int index = bp + offset;
if (ch == charAt(index)) {
return offset + bp;
}
if (ch == EOI) {
return -1;
@@ -150,14 +151,15 @@ public final class JSONReaderScanner extends JSONLexerBase {
}
if (sp > 0) {
if (this.token == JSONToken.LITERAL_STRING) {
System.arraycopy(buf, buf.length - sp, buf, 0, sp);
np = -1 + sp;
} else {
System.arraycopy(buf, bufLength - sp, buf, 0, sp);
np = 0;
int offset;
offset = bufLength - sp;
if (ch == '"') {
offset--;
}
System.arraycopy(buf, offset, buf, 0, sp);
}
np = -1;
index = bp = sp;
try {
@@ -209,7 +211,14 @@ public final class JSONReaderScanner extends JSONLexerBase {
*/
public final String stringVal() {
if (!hasSpecial) {
return new String(buf, np + 1, sp);
int offset = np + 1;
if (offset < 0) {
throw new IllegalStateException();
}
if (offset > buf.length - sp) {
throw new IllegalStateException();
}
return new String(buf, offset, sp);
// return text.substring(np + 1, np + 1 + sp);
} else {
return new String(sbuf, 0, sp);
@@ -217,19 +226,27 @@ public final class JSONReaderScanner extends JSONLexerBase {
}
public final String subString(int offset, int count) {
if (count < 0) {
throw new StringIndexOutOfBoundsException(count);
}
return new String(buf, offset, count);
// return text.substring(offset, offset + count);
}
public final String numberString() {
char chLocal = charAt(np + sp - 1);
int offset = np;
if (offset == -1) {
offset = 0;
}
char chLocal = charAt(offset + sp - 1);
int sp = this.sp;
if (chLocal == 'L' || chLocal == 'S' || chLocal == 'B' || chLocal == 'F' || chLocal == 'D') {
sp--;
}
return new String(buf, np, sp);
String value = new String(buf, offset, sp);
return value;
}
public void close() {
src/main/java/com/alibaba/fastjson/serializer/SerializeWriter.java
+59 -12
View File
@@ -511,7 +511,7 @@ public final class SerializeWriter extends Writer {
}
return;
}
writeIntAndChar(value.ordinal(), c);
}
@@ -603,10 +603,6 @@ public final class SerializeWriter extends Writer {
}
private void writeStringWithDoubleQuote(String text, final char seperator, boolean checkSpecial) {
// final boolean[] specicalFlags_doubleQuotes =
// CharTypes.specicalFlags_doubleQuotes;
// final int len_flags = specicalFlags_doubleQuotes.length;
if (text == null) {
writeNull();
if (seperator != 0) {
@@ -800,6 +796,15 @@ public final class SerializeWriter extends Writer {
for (int i = start; i < end; ++i) {
char ch = buf[i];
if (ch >= ']') {
if (ch == '\u2028') {
specialCount++;
lastSpecialIndex = i;
lastSpecial = ch;
newcount += 4;
if (firstSpecialIndex == -1) {
firstSpecialIndex = i;
}
}
continue;
}
@@ -831,9 +836,25 @@ public final class SerializeWriter extends Writer {
count = newcount;
if (specialCount == 1) {
System.arraycopy(buf, lastSpecialIndex + 1, buf, lastSpecialIndex + 2, end - lastSpecialIndex - 1);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial];
if (lastSpecial == '\u2028') {
int srcPos = lastSpecialIndex + 1;
int destPos = lastSpecialIndex + 6;
int LengthOfCopy = end - lastSpecialIndex - 1;
System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = 'u';
buf[++lastSpecialIndex] = '2';
buf[++lastSpecialIndex] = '0';
buf[++lastSpecialIndex] = '2';
buf[++lastSpecialIndex] = '8';
} else {
int srcPos = lastSpecialIndex + 1;
int destPos = lastSpecialIndex + 2;
int LengthOfCopy = end - lastSpecialIndex - 1;
System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial];
}
} else if (specialCount > 1) {
int textIndex = firstSpecialIndex - start;
int bufIndex = firstSpecialIndex;
@@ -1185,6 +1206,17 @@ public final class SerializeWriter extends Writer {
for (int i = valueStart; i < valueEnd; ++i) {
char ch = buf[i];
if (ch == '\u2028') {
specialCount++;
lastSpecialIndex = i;
lastSpecial = ch;
newcount += 4;
if (firstSpecialIndex == -1) {
firstSpecialIndex = i;
}
}
if (ch >= ']') {
continue;
}
@@ -1208,10 +1240,25 @@ public final class SerializeWriter extends Writer {
count = newcount;
if (specialCount == 1) {
System.arraycopy(buf, lastSpecialIndex + 1, buf, lastSpecialIndex + 2, valueEnd - lastSpecialIndex
- 1);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial];
if (lastSpecial == '\u2028') {
int srcPos = lastSpecialIndex + 1;
int destPos = lastSpecialIndex + 6;
int LengthOfCopy = valueEnd - lastSpecialIndex - 1;
System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = 'u';
buf[++lastSpecialIndex] = '2';
buf[++lastSpecialIndex] = '0';
buf[++lastSpecialIndex] = '2';
buf[++lastSpecialIndex] = '8';
} else {
int srcPos = lastSpecialIndex + 1;
int destPos = lastSpecialIndex + 2;
int LengthOfCopy = valueEnd - lastSpecialIndex - 1;
System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy);
buf[lastSpecialIndex] = '\\';
buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial];
}
} else if (specialCount > 1) {
int textIndex = firstSpecialIndex - valueStart;
int bufIndex = firstSpecialIndex;
src/test/java/com/alibaba/json/bvt/bug/bug_for_pengsong0302.java
+37
View File
@@ -0,0 +1,37 @@
package com.alibaba.json.bvt.bug;
import junit.framework.TestCase;
import org.junit.Assert;
import com.alibaba.fastjson.JSON;
public class bug_for_pengsong0302 extends TestCase {
public void test_0() throws Exception {
Assert.assertEquals("\"a\\u2028b\"", JSON.toJSONString("a\u2028b"));
}
public void test_1() throws Exception {
Assert.assertEquals("{\"value\":\"a\\u2028b\"}", JSON.toJSONString(new A("a\u2028b")));
}
public static class A {
private String value;
public A(String value){
super();
this.value = value;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
}
}
src/test/java/com/alibaba/json/bvt/parser/JSONReaderScannerTest__entity_double_2.java
+1 -1
View File
@@ -31,7 +31,7 @@ public class JSONReaderScannerTest__entity_double_2 extends TestCase {
DefaultJSONParser parser = new DefaultJSONParser(scanner);
List<VO> array = parser.parseArray(VO.class);
for (int i = 0; i < array.size(); ++i) {
Assert.assertTrue((double) i == array.get(i).getId());
Assert.assertTrue(Integer.toString(i), (double) i == array.get(i).getId());
}
parser.close();
}
src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string.java
-8
View File
@@ -52,12 +52,4 @@ public class JSONReader_string extends TestCase {
reader.close();
}
public void test_obj() throws Exception {
JSONReader reader = new JSONReader(new StringReader("\"abc\""));
Assert.assertEquals("abc", reader.readString());
reader.close();
}
}
src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string_1.java
+20
View File
@@ -0,0 +1,20 @@
package com.alibaba.json.bvt.parser.stream;
import java.io.StringReader;
import junit.framework.TestCase;
import org.junit.Assert;
import com.alibaba.fastjson.JSONReader;
public class JSONReader_string_1 extends TestCase {
public void test_obj() throws Exception {
JSONReader reader = new JSONReader(new StringReader("\"abc\""));
Assert.assertEquals("abc", reader.readString());
reader.close();
}
}
src/test/java/com/alibaba/json/bvt/stream/LargeTest.java
+39
View File
@@ -0,0 +1,39 @@
package com.alibaba.json.bvt.stream;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
import junit.framework.TestCase;
import org.apache.commons.io.FileUtils;
import com.alibaba.fastjson.JSONReader;
import com.alibaba.fastjson.JSONWriter;
public class LargeTest extends TestCase {
public void test_0() throws Exception {
List<String> list = new ArrayList<String>(1000 * 1);
for (int i = 0; i < 100 * 1; ++i) {
list.add(Integer.toString(i));
}
File file = File.createTempFile("fastjson-stream-large", "json");
JSONWriter writer = new JSONWriter(new FileWriter(file));
writer.startArray();
writer.writeObject(list);
writer.endArray();
writer.close();
System.out.println(FileUtils.readFileToString(file));
JSONReader reader = new JSONReader(new FileReader(file));
reader.startArray();
reader.readObject();
reader.endArray();
reader.close();
}
}