From dad4e751ec7671ec74ead496c9ebf111e640a9bb Mon Sep 17 00:00:00 2001 From: wenshao Date: Thu, 25 Jul 2013 15:28:16 +0800 Subject: [PATCH] bug fixed for stream reader api --- src/main/java/META-INF/MANIFEST.MF | 2 +- .../fastjson/parser/JSONLexerBase.java | 77 +++++++++++-------- .../fastjson/parser/JSONReaderScanner.java | 41 +++++++--- .../fastjson/serializer/SerializeWriter.java | 71 ++++++++++++++--- .../json/bvt/bug/bug_for_pengsong0302.java | 37 +++++++++ ...SONReaderScannerTest__entity_double_2.java | 2 +- .../bvt/parser/stream/JSONReader_string.java | 8 -- .../parser/stream/JSONReader_string_1.java | 20 +++++ .../alibaba/json/bvt/stream/LargeTest.java | 39 ++++++++++ 9 files changed, 232 insertions(+), 65 deletions(-) create mode 100644 src/test/java/com/alibaba/json/bvt/bug/bug_for_pengsong0302.java create mode 100644 src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string_1.java create mode 100644 src/test/java/com/alibaba/json/bvt/stream/LargeTest.java diff --git a/src/main/java/META-INF/MANIFEST.MF b/src/main/java/META-INF/MANIFEST.MF index 5e9495128..b8c791063 100755 --- a/src/main/java/META-INF/MANIFEST.MF +++ b/src/main/java/META-INF/MANIFEST.MF @@ -1,3 +1,3 @@ -Manifest-Version: 1.0 +Manifest-Version: 1.1.35 Class-Path: diff --git a/src/main/java/com/alibaba/fastjson/parser/JSONLexerBase.java b/src/main/java/com/alibaba/fastjson/parser/JSONLexerBase.java index 4a8becb2b..d9fd88e33 100755 --- a/src/main/java/com/alibaba/fastjson/parser/JSONLexerBase.java +++ b/src/main/java/com/alibaba/fastjson/parser/JSONLexerBase.java @@ -396,6 +396,9 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { public final Number integerValue() throws NumberFormatException { long result = 0; boolean negative = false; + if (np == -1) { + np = 0; + } int i = np, max = np + sp; long limit; long multmin; @@ -636,7 +639,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { boolean hasSpecial = false; char chLocal; for (;;) { - chLocal = charAt(++bp); + chLocal = next(); if (chLocal == quote) { break; @@ -665,7 +668,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { arrayCopy(np + 1, sbuf, 0, sp); } - chLocal = charAt(++bp); + chLocal = next(); switch (chLocal) { case '0': @@ -742,8 +745,8 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { putChar('\\'); break; case 'x': - char x1 = ch = charAt(++bp); - char x2 = ch = charAt(++bp); + char x1 = ch = next(); + char x2 = ch = next(); int x_val = digits[x1] * 16 + digits[x2]; char x_char = (char) x_val; @@ -751,10 +754,10 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { putChar(x_char); break; case 'u': - char c1 = chLocal = charAt(++bp); - char c2 = chLocal = charAt(++bp); - char c3 = chLocal = charAt(++bp); - char c4 = chLocal = charAt(++bp); + char c1 = chLocal = next(); + char c2 = chLocal = next(); + char c3 = chLocal = next(); + char c4 = chLocal = next(); int val = Integer.parseInt(new String(new char[] { c1, c2, c3, c4 }), 16); hash = 31 * hash + val; putChar((char) val); @@ -781,14 +784,25 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { } token = LITERAL_STRING; - this.next(); - + + String value; if (!hasSpecial) { // return this.text.substring(np + 1, np + 1 + sp).intern(); - return addSymbol(np + 1, sp, hash, symbolTable); + int offset; + if (np == -1) { + offset = 0; + } else { + offset = np + 1; + } + value = addSymbol(offset, sp, hash, symbolTable); } else { - return symbolTable.addSymbol(sbuf, 0, sp, hash); + value = symbolTable.addSymbol(sbuf, 0, sp, hash); } + + sp = 0; + this.next(); + + return value; } public final void resetStringPosition() { @@ -812,7 +826,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { sp = 1; char chLocal; for (;;) { - chLocal = charAt(++bp); + chLocal = next(); if (chLocal < identifierFlags.length) { if (!identifierFlags[chLocal]) { @@ -848,7 +862,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { hasSpecial = false; char ch; for (;;) { - ch = charAt(++bp); + ch = next(); if (ch == '\"') { break; @@ -877,7 +891,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { // System.arraycopy(buf, np + 1, sbuf, 0, sp); } - ch = charAt(++bp); + ch = next(); switch (ch) { case '0': @@ -936,18 +950,18 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { putChar('\\'); break; case 'x': - char x1 = ch = charAt(++bp); - char x2 = ch = charAt(++bp); + char x1 = ch = next(); + char x2 = ch = next(); int x_val = digits[x1] * 16 + digits[x2]; char x_char = (char) x_val; putChar(x_char); break; case 'u': - char u1 = ch = charAt(++bp); - char u2 = ch = charAt(++bp); - char u3 = ch = charAt(++bp); - char u4 = ch = charAt(++bp); + char u1 = ch = next(); + char u2 = ch = next(); + char u3 = ch = next(); + char u4 = ch = next(); int val = Integer.parseInt(new String(new char[] { u1, u2, u3, u4 }), 16); putChar((char) val); break; @@ -971,7 +985,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { } token = JSONToken.LITERAL_STRING; - this.ch = charAt(++bp); + this.ch = next(); } public Calendar getCalendar() { @@ -1151,7 +1165,8 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { throw new JSONException("unclosed str"); } - String stringVal = subString(bp + fieldName.length + 1, endIndex - startIndex); + int startIndex2 = bp + fieldName.length + 1; // must re compute + String stringVal = subString(startIndex2, endIndex - startIndex2); for (int i = bp + fieldName.length + 1; i < endIndex; ++i) { if (charAt(i) == '\\') { hasSpecial = true; @@ -2538,7 +2553,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { hasSpecial = false; char chLocal; for (;;) { - chLocal = charAt(++bp); + chLocal = next(); if (chLocal == '\'') { break; @@ -2563,7 +2578,7 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { // System.arraycopy(buf, np + 1, sbuf, 0, sp); } - chLocal = charAt(++bp); + chLocal = next(); switch (chLocal) { case '0': @@ -2622,18 +2637,18 @@ public abstract class JSONLexerBase implements JSONLexer, Closeable { putChar('\\'); break; case 'x': - char x1 = chLocal = charAt(++bp); - char x2 = chLocal = charAt(++bp); + char x1 = chLocal = next(); + char x2 = chLocal = next(); int x_val = digits[x1] * 16 + digits[x2]; char x_char = (char) x_val; putChar(x_char); break; case 'u': - char c1 = chLocal = charAt(++bp); - char c2 = chLocal = charAt(++bp); - char c3 = chLocal = charAt(++bp); - char c4 = chLocal = charAt(++bp); + char c1 = chLocal = next(); + char c2 = chLocal = next(); + char c3 = chLocal = next(); + char c4 = chLocal = next(); int val = Integer.parseInt(new String(new char[] { c1, c2, c3, c4 }), 16); putChar((char) val); break; diff --git a/src/main/java/com/alibaba/fastjson/parser/JSONReaderScanner.java b/src/main/java/com/alibaba/fastjson/parser/JSONReaderScanner.java index 54c517182..d7ed3c6c5 100644 --- a/src/main/java/com/alibaba/fastjson/parser/JSONReaderScanner.java +++ b/src/main/java/com/alibaba/fastjson/parser/JSONReaderScanner.java @@ -126,10 +126,11 @@ public final class JSONReaderScanner extends JSONLexerBase { } public final int indexOf(char ch, int startIndex) { - int offset = startIndex; + int offset = startIndex - bp; for (;; ++offset) { - if (ch == charAt(offset)) { - return offset; + final int index = bp + offset; + if (ch == charAt(index)) { + return offset + bp; } if (ch == EOI) { return -1; @@ -150,14 +151,15 @@ public final class JSONReaderScanner extends JSONLexerBase { } if (sp > 0) { - if (this.token == JSONToken.LITERAL_STRING) { - System.arraycopy(buf, buf.length - sp, buf, 0, sp); - np = -1 + sp; - } else { - System.arraycopy(buf, bufLength - sp, buf, 0, sp); - np = 0; + int offset; + offset = bufLength - sp; + if (ch == '"') { + offset--; } + System.arraycopy(buf, offset, buf, 0, sp); } + np = -1; + index = bp = sp; try { @@ -209,7 +211,14 @@ public final class JSONReaderScanner extends JSONLexerBase { */ public final String stringVal() { if (!hasSpecial) { - return new String(buf, np + 1, sp); + int offset = np + 1; + if (offset < 0) { + throw new IllegalStateException(); + } + if (offset > buf.length - sp) { + throw new IllegalStateException(); + } + return new String(buf, offset, sp); // return text.substring(np + 1, np + 1 + sp); } else { return new String(sbuf, 0, sp); @@ -217,19 +226,27 @@ public final class JSONReaderScanner extends JSONLexerBase { } public final String subString(int offset, int count) { + if (count < 0) { + throw new StringIndexOutOfBoundsException(count); + } return new String(buf, offset, count); // return text.substring(offset, offset + count); } public final String numberString() { - char chLocal = charAt(np + sp - 1); + int offset = np; + if (offset == -1) { + offset = 0; + } + char chLocal = charAt(offset + sp - 1); int sp = this.sp; if (chLocal == 'L' || chLocal == 'S' || chLocal == 'B' || chLocal == 'F' || chLocal == 'D') { sp--; } - return new String(buf, np, sp); + String value = new String(buf, offset, sp); + return value; } public void close() { diff --git a/src/main/java/com/alibaba/fastjson/serializer/SerializeWriter.java b/src/main/java/com/alibaba/fastjson/serializer/SerializeWriter.java index ed6a32109..b9baf2ba0 100755 --- a/src/main/java/com/alibaba/fastjson/serializer/SerializeWriter.java +++ b/src/main/java/com/alibaba/fastjson/serializer/SerializeWriter.java @@ -511,7 +511,7 @@ public final class SerializeWriter extends Writer { } return; } - + writeIntAndChar(value.ordinal(), c); } @@ -603,10 +603,6 @@ public final class SerializeWriter extends Writer { } private void writeStringWithDoubleQuote(String text, final char seperator, boolean checkSpecial) { - // final boolean[] specicalFlags_doubleQuotes = - // CharTypes.specicalFlags_doubleQuotes; - // final int len_flags = specicalFlags_doubleQuotes.length; - if (text == null) { writeNull(); if (seperator != 0) { @@ -800,6 +796,15 @@ public final class SerializeWriter extends Writer { for (int i = start; i < end; ++i) { char ch = buf[i]; if (ch >= ']') { + if (ch == '\u2028') { + specialCount++; + lastSpecialIndex = i; + lastSpecial = ch; + newcount += 4; + if (firstSpecialIndex == -1) { + firstSpecialIndex = i; + } + } continue; } @@ -831,9 +836,25 @@ public final class SerializeWriter extends Writer { count = newcount; if (specialCount == 1) { - System.arraycopy(buf, lastSpecialIndex + 1, buf, lastSpecialIndex + 2, end - lastSpecialIndex - 1); - buf[lastSpecialIndex] = '\\'; - buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial]; + if (lastSpecial == '\u2028') { + int srcPos = lastSpecialIndex + 1; + int destPos = lastSpecialIndex + 6; + int LengthOfCopy = end - lastSpecialIndex - 1; + System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy); + buf[lastSpecialIndex] = '\\'; + buf[++lastSpecialIndex] = 'u'; + buf[++lastSpecialIndex] = '2'; + buf[++lastSpecialIndex] = '0'; + buf[++lastSpecialIndex] = '2'; + buf[++lastSpecialIndex] = '8'; + } else { + int srcPos = lastSpecialIndex + 1; + int destPos = lastSpecialIndex + 2; + int LengthOfCopy = end - lastSpecialIndex - 1; + System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy); + buf[lastSpecialIndex] = '\\'; + buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial]; + } } else if (specialCount > 1) { int textIndex = firstSpecialIndex - start; int bufIndex = firstSpecialIndex; @@ -1185,6 +1206,17 @@ public final class SerializeWriter extends Writer { for (int i = valueStart; i < valueEnd; ++i) { char ch = buf[i]; + if (ch == '\u2028') { + specialCount++; + lastSpecialIndex = i; + lastSpecial = ch; + newcount += 4; + + if (firstSpecialIndex == -1) { + firstSpecialIndex = i; + } + } + if (ch >= ']') { continue; } @@ -1208,10 +1240,25 @@ public final class SerializeWriter extends Writer { count = newcount; if (specialCount == 1) { - System.arraycopy(buf, lastSpecialIndex + 1, buf, lastSpecialIndex + 2, valueEnd - lastSpecialIndex - - 1); - buf[lastSpecialIndex] = '\\'; - buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial]; + if (lastSpecial == '\u2028') { + int srcPos = lastSpecialIndex + 1; + int destPos = lastSpecialIndex + 6; + int LengthOfCopy = valueEnd - lastSpecialIndex - 1; + System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy); + buf[lastSpecialIndex] = '\\'; + buf[++lastSpecialIndex] = 'u'; + buf[++lastSpecialIndex] = '2'; + buf[++lastSpecialIndex] = '0'; + buf[++lastSpecialIndex] = '2'; + buf[++lastSpecialIndex] = '8'; + } else { + int srcPos = lastSpecialIndex + 1; + int destPos = lastSpecialIndex + 2; + int LengthOfCopy = valueEnd - lastSpecialIndex - 1; + System.arraycopy(buf, srcPos, buf, destPos, LengthOfCopy); + buf[lastSpecialIndex] = '\\'; + buf[++lastSpecialIndex] = replaceChars[(int) lastSpecial]; + } } else if (specialCount > 1) { int textIndex = firstSpecialIndex - valueStart; int bufIndex = firstSpecialIndex; diff --git a/src/test/java/com/alibaba/json/bvt/bug/bug_for_pengsong0302.java b/src/test/java/com/alibaba/json/bvt/bug/bug_for_pengsong0302.java new file mode 100644 index 000000000..baf178140 --- /dev/null +++ b/src/test/java/com/alibaba/json/bvt/bug/bug_for_pengsong0302.java @@ -0,0 +1,37 @@ +package com.alibaba.json.bvt.bug; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.alibaba.fastjson.JSON; + +public class bug_for_pengsong0302 extends TestCase { + + public void test_0() throws Exception { + Assert.assertEquals("\"a\\u2028b\"", JSON.toJSONString("a\u2028b")); + } + + public void test_1() throws Exception { + Assert.assertEquals("{\"value\":\"a\\u2028b\"}", JSON.toJSONString(new A("a\u2028b"))); + } + + public static class A { + + private String value; + + public A(String value){ + super(); + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + } +} diff --git a/src/test/java/com/alibaba/json/bvt/parser/JSONReaderScannerTest__entity_double_2.java b/src/test/java/com/alibaba/json/bvt/parser/JSONReaderScannerTest__entity_double_2.java index 24a509c08..b8beb2520 100644 --- a/src/test/java/com/alibaba/json/bvt/parser/JSONReaderScannerTest__entity_double_2.java +++ b/src/test/java/com/alibaba/json/bvt/parser/JSONReaderScannerTest__entity_double_2.java @@ -31,7 +31,7 @@ public class JSONReaderScannerTest__entity_double_2 extends TestCase { DefaultJSONParser parser = new DefaultJSONParser(scanner); List array = parser.parseArray(VO.class); for (int i = 0; i < array.size(); ++i) { - Assert.assertTrue((double) i == array.get(i).getId()); + Assert.assertTrue(Integer.toString(i), (double) i == array.get(i).getId()); } parser.close(); } diff --git a/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string.java b/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string.java index 32bedd931..04ca62c30 100644 --- a/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string.java +++ b/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string.java @@ -52,12 +52,4 @@ public class JSONReader_string extends TestCase { reader.close(); } - public void test_obj() throws Exception { - JSONReader reader = new JSONReader(new StringReader("\"abc\"")); - - Assert.assertEquals("abc", reader.readString()); - - reader.close(); - } - } diff --git a/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string_1.java b/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string_1.java new file mode 100644 index 000000000..dfe75b1ed --- /dev/null +++ b/src/test/java/com/alibaba/json/bvt/parser/stream/JSONReader_string_1.java @@ -0,0 +1,20 @@ +package com.alibaba.json.bvt.parser.stream; + +import java.io.StringReader; + +import junit.framework.TestCase; + +import org.junit.Assert; + +import com.alibaba.fastjson.JSONReader; + +public class JSONReader_string_1 extends TestCase { + public void test_obj() throws Exception { + JSONReader reader = new JSONReader(new StringReader("\"abc\"")); + + Assert.assertEquals("abc", reader.readString()); + + reader.close(); + } + +} diff --git a/src/test/java/com/alibaba/json/bvt/stream/LargeTest.java b/src/test/java/com/alibaba/json/bvt/stream/LargeTest.java new file mode 100644 index 000000000..00f686ddf --- /dev/null +++ b/src/test/java/com/alibaba/json/bvt/stream/LargeTest.java @@ -0,0 +1,39 @@ +package com.alibaba.json.bvt.stream; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.commons.io.FileUtils; + +import com.alibaba.fastjson.JSONReader; +import com.alibaba.fastjson.JSONWriter; + +public class LargeTest extends TestCase { + + public void test_0() throws Exception { + List list = new ArrayList(1000 * 1); + for (int i = 0; i < 100 * 1; ++i) { + list.add(Integer.toString(i)); + } + File file = File.createTempFile("fastjson-stream-large", "json"); + JSONWriter writer = new JSONWriter(new FileWriter(file)); + writer.startArray(); + writer.writeObject(list); + writer.endArray(); + writer.close(); + + System.out.println(FileUtils.readFileToString(file)); + + JSONReader reader = new JSONReader(new FileReader(file)); + reader.startArray(); + reader.readObject(); + reader.endArray(); + reader.close(); + + } +}