001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.io.input; 018 019import java.io.BufferedInputStream; 020import java.io.BufferedReader; 021import java.io.File; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.InputStreamReader; 025import java.io.Reader; 026import java.io.StringReader; 027import java.net.HttpURLConnection; 028import java.net.URL; 029import java.net.URLConnection; 030import java.nio.charset.Charset; 031import java.nio.charset.StandardCharsets; 032import java.nio.file.Files; 033import java.nio.file.Path; 034import java.text.MessageFormat; 035import java.util.Locale; 036import java.util.Objects; 037import java.util.regex.Matcher; 038import java.util.regex.Pattern; 039 040import org.apache.commons.io.ByteOrderMark; 041import org.apache.commons.io.Charsets; 042import org.apache.commons.io.IOUtils; 043import org.apache.commons.io.build.AbstractStreamBuilder; 044import org.apache.commons.io.function.IOConsumer; 045import org.apache.commons.io.output.XmlStreamWriter; 046 047/** 048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream. 049 * <p> 050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream. 051 * </p> 052 * <p> 053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100% 054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers). 055 * </p> 056 * <p> 057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors. 058 * </p> 059 * <p> 060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML 061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="http://diveintomark.org/archives/2004/02/13/xml-media-types"> Determining 062 * the character encoding of a feed</a>. 063 * </p> 064 * <p> 065 * To build an instance, see {@link Builder}. 066 * </p> 067 * <p> 068 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under Apache License 2.0. 069 * </p> 070 * 071 * @see org.apache.commons.io.output.XmlStreamWriter 072 * @since 2.0 073 */ 074public class XmlStreamReader extends Reader { 075 076 /** 077 * Builds a new {@link XmlStreamWriter} instance. 078 * 079 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 080 * <p> 081 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 082 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 083 * </p> 084 * <p> 085 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 086 * </p> 087 * <p> 088 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 089 * </p> 090 * <p> 091 * Else if the XML prolog had a charset encoding that encoding is used. 092 * </p> 093 * <p> 094 * Else if the content type had a charset encoding that encoding is used. 095 * </p> 096 * <p> 097 * Else 'UTF-8' is used. 098 * </p> 099 * <p> 100 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 101 * </p> 102 * <p> 103 * For example: 104 * </p> 105 * 106 * <pre>{@code 107 * XmlStreamReader r = XmlStreamReader.builder() 108 * .setPath(path) 109 * .setCharset(StandardCharsets.UTF_8) 110 * .get();} 111 * </pre> 112 * 113 * @since 2.12.0 114 */ 115 public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> { 116 117 private boolean nullCharset = true; 118 private boolean lenient = true; 119 private String httpContentType; 120 121 /** 122 * Constructs a new instance. 123 * <p> 124 * This builder use the aspect InputStream, OpenOption[], httpContentType, lenient, and defaultEncoding. 125 * </p> 126 * <p> 127 * You must provide an origin that can be converted to an InputStream by this builder, otherwise, this call will throw an 128 * {@link UnsupportedOperationException}. 129 * </p> 130 * 131 * @return a new instance. 132 * @throws UnsupportedOperationException if the origin cannot provide an InputStream. 133 * @throws IOException thrown if there is a problem reading the stream. 134 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 135 * @see #getInputStream() 136 */ 137 @SuppressWarnings("resource") 138 @Override 139 public XmlStreamReader get() throws IOException { 140 final String defaultEncoding = nullCharset ? null : getCharset().name(); 141 // @formatter:off 142 return httpContentType == null 143 ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding) 144 : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding); 145 // @formatter:on 146 } 147 148 @Override 149 public Builder setCharset(final Charset charset) { 150 nullCharset = charset == null; 151 return super.setCharset(charset); 152 } 153 154 @Override 155 public Builder setCharset(final String charset) { 156 nullCharset = charset == null; 157 return super.setCharset(Charsets.toCharset(charset, getCharsetDefault())); 158 } 159 160 public Builder setHttpContentType(final String httpContentType) { 161 this.httpContentType = httpContentType; 162 return this; 163 } 164 165 public Builder setLenient(final boolean lenient) { 166 this.lenient = lenient; 167 return this; 168 } 169 170 } 171 172 private static final String UTF_8 = StandardCharsets.UTF_8.name(); 173 174 private static final String US_ASCII = StandardCharsets.US_ASCII.name(); 175 176 private static final String UTF_16BE = StandardCharsets.UTF_16BE.name(); 177 178 private static final String UTF_16LE = StandardCharsets.UTF_16LE.name(); 179 180 private static final String UTF_32BE = "UTF-32BE"; 181 182 private static final String UTF_32LE = "UTF-32LE"; 183 184 private static final String UTF_16 = StandardCharsets.UTF_16.name(); 185 186 private static final String UTF_32 = "UTF-32"; 187 188 private static final String EBCDIC = "CP1047"; 189 190 private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE, 191 ByteOrderMark.UTF_32LE }; 192 193 /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */ 194 private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D), 195 new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00), 196 new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D), 197 new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00), 198 new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) }; 199 200 private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?"); 201 202 /** 203 * Pattern capturing the encoding of the "xml" processing instruction. 204 * <p> 205 * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">XML specification</a>. 206 * </p> 207 */ 208 public static final Pattern ENCODING_PATTERN = Pattern.compile( 209 // @formatter:off 210 "^<\\?xml\\s+" 211 + "version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+" 212 + "encoding\\s*=\\s*((?:\"[A-Za-z]([A-Za-z0-9\\._]|-)*\")|(?:'[A-Za-z]([A-Za-z0-9\\\\._]|-)*'))", 213 Pattern.MULTILINE); 214 // @formatter:on 215 216 private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch"; 217 218 private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM"; 219 220 private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null"; 221 222 private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch"; 223 224 private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME"; 225 226 /** 227 * Constructs a new {@link Builder}. 228 * 229 * @return a new {@link Builder}. 230 * @since 2.12.0 231 */ 232 public static Builder builder() { 233 return new Builder(); 234 } 235 236 /** 237 * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}. 238 * 239 * @param httpContentType the HTTP content type 240 * @return The content type encoding (upcased) 241 */ 242 static String getContentTypeEncoding(final String httpContentType) { 243 String encoding = null; 244 if (httpContentType != null) { 245 final int i = httpContentType.indexOf(";"); 246 if (i > -1) { 247 final String postMime = httpContentType.substring(i + 1); 248 final Matcher m = CHARSET_PATTERN.matcher(postMime); 249 encoding = m.find() ? m.group(1) : null; 250 encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null; 251 } 252 } 253 return encoding; 254 } 255 256 /** 257 * Gets the MIME type or {@code null} if httpContentType is {@code null}. 258 * 259 * @param httpContentType the HTTP content type 260 * @return The mime content type 261 */ 262 static String getContentTypeMime(final String httpContentType) { 263 String mime = null; 264 if (httpContentType != null) { 265 final int i = httpContentType.indexOf(";"); 266 if (i >= 0) { 267 mime = httpContentType.substring(0, i); 268 } else { 269 mime = httpContentType; 270 } 271 mime = mime.trim(); 272 } 273 return mime; 274 } 275 276 /** 277 * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none. 278 * 279 * @param inputStream InputStream to create the reader from. 280 * @param guessedEnc guessed encoding 281 * @return the encoding declared in the <?xml encoding=...?> 282 * @throws IOException thrown if there is a problem reading the stream. 283 */ 284 private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException { 285 String encoding = null; 286 if (guessedEnc != null) { 287 final byte[] bytes = IOUtils.byteArray(); 288 inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE); 289 int offset = 0; 290 int max = IOUtils.DEFAULT_BUFFER_SIZE; 291 int c = inputStream.read(bytes, offset, max); 292 int firstGT = -1; 293 String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning) 294 while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) { 295 offset += c; 296 max -= c; 297 c = inputStream.read(bytes, offset, max); 298 xmlProlog = new String(bytes, 0, offset, guessedEnc); 299 firstGT = xmlProlog.indexOf('>'); 300 } 301 if (firstGT == -1) { 302 if (c == -1) { 303 throw new IOException("Unexpected end of XML stream"); 304 } 305 throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes"); 306 } 307 final int bytesRead = offset; 308 if (bytesRead > 0) { 309 inputStream.reset(); 310 final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1))); 311 final StringBuilder prolog = new StringBuilder(); 312 IOConsumer.forEach(bReader.lines(), prolog::append); 313 final Matcher m = ENCODING_PATTERN.matcher(prolog); 314 if (m.find()) { 315 encoding = m.group(1).toUpperCase(Locale.ROOT); 316 encoding = encoding.substring(1, encoding.length() - 1); 317 } 318 } 319 } 320 return encoding; 321 } 322 323 /** 324 * Tests if the MIME type belongs to the APPLICATION XML family. 325 * 326 * @param mime The mime type 327 * @return true if the mime type belongs to the APPLICATION XML family, otherwise false 328 */ 329 static boolean isAppXml(final String mime) { 330 return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity") 331 || mime.startsWith("application/") && mime.endsWith("+xml")); 332 } 333 334 /** 335 * Tests if the MIME type belongs to the TEXT XML family. 336 * 337 * @param mime The mime type 338 * @return true if the mime type belongs to the TEXT XML family, otherwise false 339 */ 340 static boolean isTextXml(final String mime) { 341 return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml")); 342 } 343 344 private final Reader reader; 345 346 private final String encoding; 347 348 private final String defaultEncoding; 349 350 /** 351 * Constructs a Reader for a File. 352 * <p> 353 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 354 * </p> 355 * <p> 356 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 357 * </p> 358 * 359 * @param file File to create a Reader from. 360 * @throws NullPointerException if the input is {@code null}. 361 * @throws IOException thrown if there is a problem reading the file. 362 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 363 */ 364 @Deprecated 365 public XmlStreamReader(final File file) throws IOException { 366 this(Objects.requireNonNull(file, "file").toPath()); 367 } 368 369 /** 370 * Constructs a Reader for a raw InputStream. 371 * <p> 372 * It follows the same logic used for files. 373 * </p> 374 * <p> 375 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 376 * </p> 377 * 378 * @param inputStream InputStream to create a Reader from. 379 * @throws NullPointerException if the input stream is {@code null}. 380 * @throws IOException thrown if there is a problem reading the stream. 381 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 382 */ 383 @Deprecated 384 public XmlStreamReader(final InputStream inputStream) throws IOException { 385 this(inputStream, true); 386 } 387 388 /** 389 * Constructs a Reader for a raw InputStream. 390 * <p> 391 * It follows the same logic used for files. 392 * </p> 393 * <p> 394 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 395 * </p> 396 * <p> 397 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 398 * </p> 399 * <p> 400 * Else if the XML prolog had a charset encoding that encoding is used. 401 * </p> 402 * <p> 403 * Else if the content type had a charset encoding that encoding is used. 404 * </p> 405 * <p> 406 * Else 'UTF-8' is used. 407 * </p> 408 * <p> 409 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 410 * </p> 411 * 412 * @param inputStream InputStream to create a Reader from. 413 * @param lenient indicates if the charset encoding detection should be relaxed. 414 * @throws NullPointerException if the input stream is {@code null}. 415 * @throws IOException thrown if there is a problem reading the stream. 416 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 417 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 418 */ 419 @Deprecated 420 public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException { 421 this(inputStream, lenient, null); 422 } 423 424 /** 425 * Constructs a Reader for a raw InputStream. 426 * <p> 427 * It follows the same logic used for files. 428 * </p> 429 * <p> 430 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 431 * </p> 432 * <p> 433 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 434 * </p> 435 * <p> 436 * Else if the XML prolog had a charset encoding that encoding is used. 437 * </p> 438 * <p> 439 * Else if the content type had a charset encoding that encoding is used. 440 * </p> 441 * <p> 442 * Else 'UTF-8' is used. 443 * </p> 444 * <p> 445 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 446 * </p> 447 * 448 * @param inputStream InputStream to create a Reader from. 449 * @param lenient indicates if the charset encoding detection should be relaxed. 450 * @param defaultEncoding The default encoding 451 * @throws NullPointerException if the input stream is {@code null}. 452 * @throws IOException thrown if there is a problem reading the stream. 453 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 454 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 455 */ 456 @Deprecated 457 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 458 public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException { 459 Objects.requireNonNull(inputStream, "inputStream"); 460 this.defaultEncoding = defaultEncoding; 461 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS); 462 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 463 this.encoding = doRawStream(bom, pis, lenient); 464 this.reader = new InputStreamReader(pis, encoding); 465 } 466 467 /** 468 * Constructs a Reader using an InputStream and the associated content-type header. 469 * <p> 470 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 471 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 472 * </p> 473 * <p> 474 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 475 * </p> 476 * 477 * @param inputStream InputStream to create the reader from. 478 * @param httpContentType content-type header to use for the resolution of the charset encoding. 479 * @throws NullPointerException if the input stream is {@code null}. 480 * @throws IOException thrown if there is a problem reading the file. 481 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 482 */ 483 @Deprecated 484 public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException { 485 this(inputStream, httpContentType, true); 486 } 487 488 /** 489 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 490 * <p> 491 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 492 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 493 * </p> 494 * <p> 495 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 496 * </p> 497 * <p> 498 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 499 * </p> 500 * <p> 501 * Else if the XML prolog had a charset encoding that encoding is used. 502 * </p> 503 * <p> 504 * Else if the content type had a charset encoding that encoding is used. 505 * </p> 506 * <p> 507 * Else 'UTF-8' is used. 508 * </p> 509 * <p> 510 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 511 * </p> 512 * 513 * @param inputStream InputStream to create the reader from. 514 * @param httpContentType content-type header to use for the resolution of the charset encoding. 515 * @param lenient indicates if the charset encoding detection should be relaxed. 516 * @throws NullPointerException if the input stream is {@code null}. 517 * @throws IOException thrown if there is a problem reading the file. 518 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 519 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 520 */ 521 @Deprecated 522 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException { 523 this(inputStream, httpContentType, lenient, null); 524 } 525 526 /** 527 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 528 * <p> 529 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 530 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 531 * </p> 532 * <p> 533 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 534 * </p> 535 * <p> 536 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 537 * </p> 538 * <p> 539 * Else if the XML prolog had a charset encoding that encoding is used. 540 * </p> 541 * <p> 542 * Else if the content type had a charset encoding that encoding is used. 543 * </p> 544 * <p> 545 * Else 'UTF-8' is used. 546 * </p> 547 * <p> 548 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 549 * </p> 550 * 551 * @param inputStream InputStream to create the reader from. 552 * @param httpContentType content-type header to use for the resolution of the charset encoding. 553 * @param lenient indicates if the charset encoding detection should be relaxed. 554 * @param defaultEncoding The default encoding 555 * @throws NullPointerException if the input stream is {@code null}. 556 * @throws IOException thrown if there is a problem reading the file. 557 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 558 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 559 */ 560 @Deprecated 561 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 562 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding) 563 throws IOException { 564 Objects.requireNonNull(inputStream, "inputStream"); 565 this.defaultEncoding = defaultEncoding; 566 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS); 567 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 568 this.encoding = processHttpStream(bom, pis, httpContentType, lenient); 569 this.reader = new InputStreamReader(pis, encoding); 570 } 571 572 /** 573 * Constructs a Reader for a File. 574 * <p> 575 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 576 * </p> 577 * <p> 578 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 579 * </p> 580 * 581 * @param file File to create a Reader from. 582 * @throws NullPointerException if the input is {@code null}. 583 * @throws IOException thrown if there is a problem reading the file. 584 * @since 2.11.0 585 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 586 */ 587 @Deprecated 588 @SuppressWarnings("resource") // InputStream is managed through another reader in this instance. 589 public XmlStreamReader(final Path file) throws IOException { 590 this(Files.newInputStream(Objects.requireNonNull(file, "file"))); 591 } 592 593 /** 594 * Constructs a Reader using the InputStream of a URL. 595 * <p> 596 * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files. 597 * </p> 598 * <p> 599 * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type. 600 * </p> 601 * <p> 602 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 603 * </p> 604 * 605 * @param url URL to create a Reader from. 606 * @throws NullPointerException if the input is {@code null}. 607 * @throws IOException thrown if there is a problem reading the stream of the URL. 608 */ 609 public XmlStreamReader(final URL url) throws IOException { 610 this(Objects.requireNonNull(url, "url").openConnection(), null); 611 } 612 613 /** 614 * Constructs a Reader using the InputStream of a URLConnection. 615 * <p> 616 * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files. 617 * </p> 618 * <p> 619 * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with 620 * content-type. 621 * </p> 622 * <p> 623 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 624 * </p> 625 * 626 * @param urlConnection URLConnection to create a Reader from. 627 * @param defaultEncoding The default encoding 628 * @throws NullPointerException if the input is {@code null}. 629 * @throws IOException thrown if there is a problem reading the stream of the URLConnection. 630 */ 631 public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException { 632 Objects.requireNonNull(urlConnection, "urlConnection"); 633 this.defaultEncoding = defaultEncoding; 634 final boolean lenient = true; 635 final String contentType = urlConnection.getContentType(); 636 final InputStream inputStream = urlConnection.getInputStream(); 637 @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance 638 // @formatter:off 639 final BOMInputStream bomInput = BOMInputStream.builder() 640 .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE)) 641 .setInclude(false) 642 .setByteOrderMarks(BOMS) 643 .get(); 644 @SuppressWarnings("resource") 645 final BOMInputStream piInput = BOMInputStream.builder() 646 .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE)) 647 .setInclude(true) 648 .setByteOrderMarks(XML_GUESS_BYTES) 649 .get(); 650 // @formatter:on 651 if (urlConnection instanceof HttpURLConnection || contentType != null) { 652 this.encoding = processHttpStream(bomInput, piInput, contentType, lenient); 653 } else { 654 this.encoding = doRawStream(bomInput, piInput, lenient); 655 } 656 this.reader = new InputStreamReader(piInput, encoding); 657 } 658 659 /** 660 * Calculates the HTTP encoding. 661 * 662 * @param httpContentType The HTTP content type 663 * @param bomEnc BOM encoding 664 * @param xmlGuessEnc XML Guess encoding 665 * @param xmlEnc XML encoding 666 * @param lenient indicates if the charset encoding detection should be relaxed. 667 * @return the HTTP encoding 668 * @throws IOException thrown if there is a problem reading the stream. 669 */ 670 String calculateHttpEncoding(final String httpContentType, final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient) 671 throws IOException { 672 673 // Lenient and has XML encoding 674 if (lenient && xmlEnc != null) { 675 return xmlEnc; 676 } 677 678 // Determine mime/encoding content types from HTTP Content Type 679 final String cTMime = getContentTypeMime(httpContentType); 680 final String cTEnc = getContentTypeEncoding(httpContentType); 681 final boolean appXml = isAppXml(cTMime); 682 final boolean textXml = isTextXml(cTMime); 683 684 // Mime type NOT "application/xml" or "text/xml" 685 if (!appXml && !textXml) { 686 final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 687 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 688 } 689 690 // No content type encoding 691 if (cTEnc == null) { 692 if (appXml) { 693 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 694 } 695 return defaultEncoding == null ? US_ASCII : defaultEncoding; 696 } 697 698 // UTF-16BE or UTF-16LE content type encoding 699 if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) { 700 if (bomEnc != null) { 701 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 702 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 703 } 704 return cTEnc; 705 } 706 707 // UTF-16 content type encoding 708 if (cTEnc.equals(UTF_16)) { 709 if (bomEnc != null && bomEnc.startsWith(UTF_16)) { 710 return bomEnc; 711 } 712 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 713 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 714 } 715 716 // UTF-32BE or UTF-132E content type encoding 717 if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) { 718 if (bomEnc != null) { 719 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 720 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 721 } 722 return cTEnc; 723 } 724 725 // UTF-32 content type encoding 726 if (cTEnc.equals(UTF_32)) { 727 if (bomEnc != null && bomEnc.startsWith(UTF_32)) { 728 return bomEnc; 729 } 730 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 731 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 732 } 733 734 return cTEnc; 735 } 736 737 /** 738 * Calculate the raw encoding. 739 * 740 * @param bomEnc BOM encoding 741 * @param xmlGuessEnc XML Guess encoding 742 * @param xmlEnc XML encoding 743 * @return the raw encoding 744 * @throws IOException thrown if there is a problem reading the stream. 745 */ 746 String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException { 747 748 // BOM is Null 749 if (bomEnc == null) { 750 if (xmlGuessEnc == null || xmlEnc == null) { 751 return defaultEncoding == null ? UTF_8 : defaultEncoding; 752 } 753 if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) { 754 return xmlGuessEnc; 755 } 756 return xmlEnc; 757 } 758 759 // BOM is UTF-8 760 if (bomEnc.equals(UTF_8)) { 761 if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) { 762 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 763 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 764 } 765 if (xmlEnc != null && !xmlEnc.equals(UTF_8)) { 766 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 767 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 768 } 769 return bomEnc; 770 } 771 772 // BOM is UTF-16BE or UTF-16LE 773 if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) { 774 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 775 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 776 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 777 } 778 if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) { 779 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 780 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 781 } 782 return bomEnc; 783 } 784 785 // BOM is UTF-32BE or UTF-32LE 786 if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) { 787 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 788 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 789 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 790 } 791 if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) { 792 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 793 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 794 } 795 return bomEnc; 796 } 797 798 // BOM is something else 799 final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc); 800 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 801 } 802 803 /** 804 * Closes the XmlStreamReader stream. 805 * 806 * @throws IOException thrown if there was a problem closing the stream. 807 */ 808 @Override 809 public void close() throws IOException { 810 reader.close(); 811 } 812 813 /** 814 * Does lenient detection. 815 * 816 * @param httpContentType content-type header to use for the resolution of the charset encoding. 817 * @param ex The thrown exception 818 * @return the encoding 819 * @throws IOException thrown if there is a problem reading the stream. 820 */ 821 private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException { 822 if (httpContentType != null && httpContentType.startsWith("text/html")) { 823 httpContentType = httpContentType.substring("text/html".length()); 824 httpContentType = "text/xml" + httpContentType; 825 try { 826 return calculateHttpEncoding(httpContentType, ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true); 827 } catch (final XmlStreamReaderException ex2) { 828 ex = ex2; 829 } 830 } 831 String encoding = ex.getXmlEncoding(); 832 if (encoding == null) { 833 encoding = ex.getContentTypeEncoding(); 834 } 835 if (encoding == null) { 836 encoding = defaultEncoding == null ? UTF_8 : defaultEncoding; 837 } 838 return encoding; 839 } 840 841 /** 842 * Process the raw stream. 843 * 844 * @param bom BOMInputStream to detect byte order marks 845 * @param pis BOMInputStream to guess XML encoding 846 * @param lenient indicates if the charset encoding detection should be relaxed. 847 * @return the encoding to be used 848 * @throws IOException thrown if there is a problem reading the stream. 849 */ 850 private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient) throws IOException { 851 final String bomEnc = bom.getBOMCharsetName(); 852 final String xmlGuessEnc = pis.getBOMCharsetName(); 853 final String xmlEnc = getXmlProlog(pis, xmlGuessEnc); 854 try { 855 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 856 } catch (final XmlStreamReaderException ex) { 857 if (lenient) { 858 return doLenientDetection(null, ex); 859 } 860 throw ex; 861 } 862 } 863 864 /** 865 * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate. 866 * <p> 867 * If it is {@code null} the content-type based rules are used. 868 * </p> 869 * 870 * @return the default encoding to use. 871 */ 872 public String getDefaultEncoding() { 873 return defaultEncoding; 874 } 875 876 /** 877 * Gets the charset encoding of the XmlStreamReader. 878 * 879 * @return charset encoding. 880 */ 881 public String getEncoding() { 882 return encoding; 883 } 884 885 /** 886 * Processes an HTTP stream. 887 * 888 * @param bomInput BOMInputStream to detect byte order marks 889 * @param piInput BOMInputStream to guess XML encoding 890 * @param httpContentType The HTTP content type 891 * @param lenient indicates if the charset encoding detection should be relaxed. 892 * @return the encoding to be used 893 * @throws IOException thrown if there is a problem reading the stream. 894 */ 895 private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final String httpContentType, final boolean lenient) 896 throws IOException { 897 final String bomEnc = bomInput.getBOMCharsetName(); 898 final String xmlGuessEnc = piInput.getBOMCharsetName(); 899 final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc); 900 try { 901 return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient); 902 } catch (final XmlStreamReaderException ex) { 903 if (lenient) { 904 return doLenientDetection(httpContentType, ex); 905 } 906 throw ex; 907 } 908 } 909 910 /** 911 * Reads the underlying reader's {@code read(char[], int, int)} method. 912 * 913 * @param buf the buffer to read the characters into 914 * @param offset The start offset 915 * @param len The number of bytes to read 916 * @return the number of characters read or -1 if the end of stream 917 * @throws IOException if an I/O error occurs. 918 */ 919 @Override 920 public int read(final char[] buf, final int offset, final int len) throws IOException { 921 return reader.read(buf, offset, len); 922 } 923 924}