initial/final commitHEADmaster

1 files changed, 252 insertions, 0 deletions

diff --git a/vmime-master/tests/parser/charsetTest.cpp b/vmime-master/tests/parser/charsetTest.cpp
new file mode 100644
index 0000000..e44bef5
--- /dev/null
+++ b/vmime-master/tests/parser/charsetTest.cpp
@@ -0,0 +1,252 @@
+//
+// VMime library (http://www.vmime.org)
+// Copyright (C) 2002 Vincent Richard <vincent@vmime.org>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 3 of
+// the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Linking this library statically or dynamically with other modules is making
+// a combined work based on this library.  Thus, the terms and conditions of
+// the GNU General Public License cover the whole combination.
+//
+
+#include <algorithm>
+
+#include "tests/testUtils.hpp"
+
+#include "charsetTestSuites.hpp"
+
+
+VMIME_TEST_SUITE_BEGIN(charsetTest)
+
+	VMIME_TEST_LIST_BEGIN
+		// Test valid input
+		VMIME_TEST(testConvertStringValid)
+		VMIME_TEST(testConvertStreamValid)
+		VMIME_TEST(testEncodingHebrew1255)
+
+		// IDNA
+		VMIME_TEST(testEncodeIDNA)
+		VMIME_TEST(testDecodeIDNA)
+
+		VMIME_TEST(testUTF7Support)
+
+		VMIME_TEST(testReplaceInvalidSequence)
+		VMIME_TEST(testStopOnInvalidSequence)
+
+		VMIME_TEST(testStatus)
+		VMIME_TEST(testStatusWithInvalidSequence)
+
+		VMIME_TEST(testIsValidText)
+	VMIME_TEST_LIST_END
+
+
+	void testConvertStringValid() {
+
+		for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) {
+
+			const charsetTestSuiteStruct& entry = charsetTestSuites[i];
+
+			std::ostringstream testName;
+			testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset;
+
+			const unsigned long inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength);
+			vmime::string in(entry.fromBytes, entry.fromBytes + inLength);
+
+			const unsigned long outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength);
+			vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength);
+
+			vmime::string actualOut;
+
+			vmime::charset::convert
+				(in, actualOut, entry.fromCharset, entry.toCharset);
+
+			VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut));
+		}
+	}
+
+	void testConvertStreamValid() {
+
+		for (unsigned int i = 0 ; i < charsetTestSuitesCount ; ++i) {
+
+			const charsetTestSuiteStruct& entry = charsetTestSuites[i];
+
+			std::ostringstream testName;
+			testName << i << ": " << entry.fromCharset << " -> " << entry.toCharset;
+
+			const unsigned long inLength = (entry.fromLength == 0 ? strlen(entry.fromBytes) : entry.fromLength);
+			vmime::string in(entry.fromBytes, entry.fromBytes + inLength);
+
+			const unsigned long outLength = (entry.toLength == 0 ? strlen(entry.toBytes) : entry.toLength);
+			vmime::string expectedOut(entry.toBytes, entry.toBytes + outLength);
+
+			vmime::string actualOut;
+			vmime::utility::outputStreamStringAdapter os(actualOut);
+
+			vmime::utility::inputStreamStringAdapter is(in);
+
+			vmime::charset::convert(is, os, entry.fromCharset, entry.toCharset);
+
+			os.flush();
+
+			VASSERT_EQ(testName.str(), toHex(expectedOut), toHex(actualOut));
+		}
+	}
+
+	void testEncodingHebrew1255() {
+
+		// hewbrew string in windows-1255 charset
+		const char data[] = "\xe9\xf9\xf7\xf8\xe9\xf9\xf8\xf7\xe9\xe9\xf9";
+		vmime::word w = vmime::word(data, "windows-1255");
+		vmime::string encoded = w.generate();
+		// less than 60% ascii, base64 received
+		VASSERT_EQ("1", "=?windows-1255?B?6fn3+On5+Pfp6fk=?=", encoded);
+	}
+
+	static const vmime::string convertHelper(
+		const vmime::string& in,
+		const vmime::charset& csrc,
+		const vmime::charset& cdest,
+		const vmime::charsetConverterOptions& opts = vmime::charsetConverterOptions(),
+		vmime::charsetConverter::status* st = NULL
+	) {
+
+		vmime::shared_ptr <vmime::charsetConverter> conv =
+			vmime::charsetConverter::create(csrc, cdest, opts);
+
+		vmime::string out;
+		conv->convert(in, out, st);
+
+		return out;
+	}
+
+	void testEncodeIDNA() {
+
+		VASSERT_EQ("1", "xn--espaol-zwa", convertHelper("español", "utf-8", "idna"));
+
+		// Tests from ICANN
+		VASSERT_EQ("2.1", "xn--hxajbheg2az3al", convertHelper("παράδειγμα", "utf-8", "idna"));
+		VASSERT_EQ("2.2", "xn--jxalpdlp", convertHelper("δοκιμή", "utf-8", "idna"));
+
+		VASSERT_EQ("3.1", "xn--mgbh0fb", convertHelper("مثال", "utf-8", "idna"));
+		VASSERT_EQ("3.2", "xn--kgbechtv", convertHelper("إختبار", "utf-8", "idna"));
+	}
+
+	void testDecodeIDNA() {
+
+		VASSERT_EQ("1", "español", convertHelper("xn--espaol-zwa", "idna", "utf-8"));
+
+		// Tests from ICANN
+		VASSERT_EQ("2.1", "παράδειγμα", convertHelper("xn--hxajbheg2az3al", "idna", "utf-8"));
+		VASSERT_EQ("2.2", "δοκιμή", convertHelper("xn--jxalpdlp", "idna", "utf-8"));
+
+		VASSERT_EQ("3.1", "مثال", convertHelper("xn--mgbh0fb", "idna", "utf-8"));
+		VASSERT_EQ("3.2", "إختبار", convertHelper("xn--kgbechtv", "idna", "utf-8"));
+	}
+
+	void testUTF7Support() {
+
+		// Ensure UTF-7 is supported, because it is used for IMAP
+		VASSERT_EQ("1", "VMime +- UTF-7 encoding", convertHelper("VMime + UTF-7 encoding", "utf-8", "utf-7"));
+		VASSERT_EQ("2", "f+APg-o", convertHelper("\x66\xc3\xb8\x6f", "utf-8", "utf-7"));
+	}
+
+	void testReplaceInvalidSequence() {
+
+		vmime::charsetConverterOptions opts;
+		opts.silentlyReplaceInvalidSequences = true;
+		opts.invalidSequence = "?";
+
+		vmime::string res = convertHelper(
+			"\x61\xf1\x80\x80\xe1\x80\xc2\x62\x80\x63\x80\xbf\x64",
+			"utf-8", "iso-8859-1", opts
+		);
+
+		// Result should be in the form "a???b?c??d" or "a??????b?c??d"...
+		// Remove consecutive question marks for easier matching.
+		res.erase(std::unique(res.begin(), res.end()), res.end());
+
+		VASSERT_EQ(
+			"Illegal UTF-8 sequence",
+			"a?b?c?d",
+			res
+		);
+	}
+
+	void testStopOnInvalidSequence() {
+
+		vmime::charsetConverterOptions opts;
+		opts.silentlyReplaceInvalidSequences = false;
+
+		VASSERT_THROW(
+			"Illegal UTF-8 sequence",
+			convertHelper("\x61\xf1\x80\x80\xe1\x80\xc2\x62\x80\x63\x80\xbf\x64", "utf-8", "iso-8859-1", opts),
+			vmime::exceptions::illegal_byte_sequence_for_charset
+		);
+	}
+
+	void testStatus() {
+
+		vmime::charsetConverterOptions opts;
+		opts.silentlyReplaceInvalidSequences = false;
+
+		vmime::charsetConverter::status st;
+
+		//             012345   6   7
+		convertHelper("Gwena\xc3\xabl", "utf-8", "iso-8859-1", opts, &st);
+
+		VASSERT_EQ("inputBytesRead", 8, st.inputBytesRead);
+		VASSERT_EQ("outputBytesWritten", 7, st.outputBytesWritten);
+	}
+
+	void testStatusWithInvalidSequence() {
+
+		vmime::charsetConverterOptions opts;
+		opts.silentlyReplaceInvalidSequences = false;
+
+		vmime::charsetConverter::status st;
+
+		try {
+
+			//             01234   5   6789   0   1
+			convertHelper("Fran\xc3\xa7ois\xf1\x80\x65", "utf-8", "iso-8859-1", opts, &st);
+
+		} catch (vmime::exceptions::illegal_byte_sequence_for_charset& e) {
+
+		} catch (...) {
+
+			throw;
+		}
+
+		VASSERT_EQ("inputBytesRead", 9, st.inputBytesRead);
+		VASSERT_EQ("outputBytesWritten", 8, st.outputBytesWritten);
+	}
+
+	void testIsValidText() {
+
+		// Invalid text
+		const vmime::string invalidText("Fran\xc3\xa7ois\xf1\x80\x65");
+		vmime::string::size_type firstInvalidByte;
+
+		VASSERT_EQ("invalid.isValidText", false, vmime::charset("utf-8").isValidText(invalidText, &firstInvalidByte));
+		VASSERT_EQ("invalid.firstInvalidByte", 9, firstInvalidByte);
+
+		// Valid text
+		const vmime::string validText("Gwena\xc3\xabl");
+
+		VASSERT_EQ("valid.isValidText", true, vmime::charset("utf-8").isValidText(validText, &firstInvalidByte));
+		VASSERT_EQ("valid.firstInvalidByte", 8, firstInvalidByte);
+	}
+
+VMIME_TEST_SUITE_END