0

我正在读取 Shift-JIS 编码的 XML 文件并将其存储在 ByteBuffer 中，然后将其转换为字符串并尝试通过 Pattern & Matcher 查找字符串的开头和字符串的结尾。从这两个位置我尝试将缓冲区写入文件。它在没有多字节字符时工作。如果有一个多字节字符，我会在最后错过一些文本，因为 end 的值很少

static final Pattern startPattern = Pattern.compile("<\\?xml ");
static final Pattern endPattern = Pattern.compile("</doc>\n");

 public static void main(String[] args) throws Exception {
    File f = new File("20121114000606JA.xml");
    FileInputStream fis = new FileInputStream(f);
    FileChannel fci = fis.getChannel();
    ByteBuffer data_buffer = ByteBuffer.allocate(65536);
    while (true) {
      int read = fci.read(data_buffer);
      if (read == -1)
        break;
    }

    ByteBuffer cbytes = data_buffer.duplicate();
    cbytes.flip();
    Charset data_charset = Charset.forName("UTF-8");
    String request = data_charset.decode(cbytes).toString();

    Matcher start = startPattern.matcher(request);
    if (start.find()) {
      Matcher end = endPattern.matcher(request);

      if (end.find()) {

        int i0 = start.start();
        int i1 = end.end();

        String str = request.substring(i0, i1);

        String filename = "test.xml";
        FileChannel fc = new FileOutputStream(new File(filename), false).getChannel();

        data_buffer.position(i0);
        data_buffer.limit(i1 - i0);

        long offset = fc.position();
        long sz = fc.write(data_buffer);

        fc.close();
      }
    }
    System.out.println("OK");
  }
4

3 回答 3

1

使用字符串索引i0 和 i1 作为字节位置：

data_buffer.position(i0);
data_buffer.limit(i1 - i0);

是错误的。由于 UTF-8 没有给出唯一的编码，ĉ被写为两个字符c+ 组合变音标记^，字符和字节之间的来回转换不仅昂贵而且容易出错（在特定数据的 rand 情况下）。

BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
        new File(filename)), "UTF-8"));

或者使用实现 CharSequence 的 CharBuffer。

而不是写入 FileChannel fc：

BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
        new File(filename)), "UTF-8"));
try {
    out.write(str);
} finally {
    out.close();
}

CharBuffer 版本需要更多的重写，也涉及到模式匹配。

于 2012-11-15T15:14:28.543 回答
0

您的问题似乎与您对字节缓冲区的解码有关。您正在使用 UTF-8 CharSet 解码 Shift-JIS ByteBuffer。您需要将其更改为 Shift-JIS CharSet。这些是受支持的字符编码。

虽然我没有要测试的 Shift-JIS 文件，但您应该尝试将 CharSet.forName 行更改为：

Charset data_charset = Charset.forName("Shift_JIS");

此外，您的正则表达式逻辑有点偏离。我不会使用第二个匹配器，因为这会导致搜索重新开始并可能导致范围反转。相反，尝试获取当前匹配的位置，然后更改匹配器正在使用的模式：

Matcher matcher = startPattern.matcher(request);
if (matcher.find()) {
  int i0 = matcher.start();
  matcher.usePattern(endPattern);

  if (matcher.find()) {

    int i1 = matcher.end();

由于 Shift-JIS 是一个两字节编码系统，它应该干净地映射到 Java UTF-8 字符。这应该允许您将其与“START.*END”之类的单个模式匹配，并且只需使用组来获取您的数据。

于 2012-11-15T17:54:45.700 回答
0

要正确转码此文件，您应该使用 Java 的 XML API。虽然有几种方法可以做到这一点，但这里有一个使用 javax.xml.transform 包的解决方案。首先，我们确实需要文档中引用的 djnml-1.0b.dtd 文件（以防它包含实体引用。）由于缺少此文件，因此此解决方案使用从提供的输入生成的 DTD，使用Trang：

<?xml encoding="UTF-8"?>

<!ELEMENT doc (djnml)>
<!ATTLIST doc
  xmlns CDATA #FIXED ''
  destination NMTOKEN #REQUIRED
  distId NMTOKEN #REQUIRED
  md5 CDATA #REQUIRED
  msize CDATA #REQUIRED
  sysId NMTOKEN #REQUIRED
  transmission-date NMTOKEN #REQUIRED>

<!ELEMENT djnml (head,body)>
<!ATTLIST djnml
  xmlns CDATA #FIXED ''
  docdate CDATA #REQUIRED
  product NMTOKEN #REQUIRED
  publisher NMTOKEN #REQUIRED
  seq CDATA #REQUIRED
  xml:lang NMTOKEN #REQUIRED>

<!ELEMENT head (copyright,docdata)>
<!ATTLIST head
  xmlns CDATA #FIXED ''>

<!ELEMENT body (headline,text)>
<!ATTLIST body
  xmlns CDATA #FIXED ''>

<!ELEMENT copyright EMPTY>
<!ATTLIST copyright
  xmlns CDATA #FIXED ''
  holder CDATA #REQUIRED
  year CDATA #REQUIRED>

<!ELEMENT docdata (djn)>
<!ATTLIST docdata
  xmlns CDATA #FIXED ''>

<!ELEMENT headline (#PCDATA)>
<!ATTLIST headline
  xmlns CDATA #FIXED ''
  brand-display NMTOKEN #REQUIRED
  prefix CDATA #REQUIRED>

<!ELEMENT text (pre,p+)>
<!ATTLIST text
  xmlns CDATA #FIXED ''>

<!ELEMENT djn (djn-newswires)>
<!ATTLIST djn
  xmlns CDATA #FIXED ''>

<!ELEMENT pre EMPTY>
<!ATTLIST pre
  xmlns CDATA #FIXED ''>

<!ELEMENT p (#PCDATA)>
<!ATTLIST p
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-newswires (djn-press-cutout,djn-urgency,djn-mdata)>
<!ATTLIST djn-newswires
  xmlns CDATA #FIXED ''
  news-source NMTOKEN #REQUIRED
  origin NMTOKEN #REQUIRED
  service-id NMTOKEN #REQUIRED>

<!ELEMENT djn-press-cutout EMPTY>
<!ATTLIST djn-press-cutout
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-urgency (#PCDATA)>
<!ATTLIST djn-urgency
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-mdata (djn-coding)>
<!ATTLIST djn-mdata
  xmlns CDATA #FIXED ''
  accession-number CDATA #REQUIRED
  brand NMTOKEN #REQUIRED
  display-date NMTOKEN #REQUIRED
  hot NMTOKEN #REQUIRED
  original-source NMTOKEN #REQUIRED
  page-citation CDATA #REQUIRED
  retention NMTOKEN #REQUIRED
  temp-perm NMTOKEN #REQUIRED>

<!ELEMENT djn-coding (djn-company,djn-isin,djn-industry,djn-subject,
                      djn-market,djn-product,djn-geo)>
<!ATTLIST djn-coding
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-company (c)>
<!ATTLIST djn-company
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-isin (c)>
<!ATTLIST djn-isin
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-industry (c)+>
<!ATTLIST djn-industry
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-subject (c)+>
<!ATTLIST djn-subject
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-market (c)+>
<!ATTLIST djn-market
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-product (c)+>
<!ATTLIST djn-product
  xmlns CDATA #FIXED ''>

<!ELEMENT djn-geo (c)+>
<!ATTLIST djn-geo
  xmlns CDATA #FIXED ''>

<!ELEMENT c (#PCDATA)>
<!ATTLIST c
  xmlns CDATA #FIXED ''>

将此文件写入“djnml-1.0b.dtd”后，我们需要使用 XSLT 创建一个身份转换。您可以使用 TransformerFactory 上的 newTransformer() 方法来执行此操作，但此转换的结果没有很好地指定。使用 XSLT 将产生更清晰的结果。我们将使用这个文件作为我们的身份转换：

<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

  <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes" omit-xml-declaration="no"/>

  <xsl:template match="@*|node()">
    <xsl:copy>
      <xsl:apply-templates select="@*|node()"/>
    </xsl:copy>
  </xsl:template>

</xsl:stylesheet>

将上述 XSLT 文件另存为“identity.xsl”。现在我们有了 DTD 和身份转换，我们可以使用以下代码对文件进行转码：

import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;

import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.XMLReaderFactory;

...

File inFile = new File("20121114000606JA.xml");
File outputFile = new File("test.xml");
final File dtdFile = new File("djnml-1.0b.dtd");
File identityFile = new File("identity.xsl");

final List<Closeable> closeables = new ArrayList<Closeable>();
try {
  // We are going to use a SAXSource for input, so that we can specify the
  // location of the DTD with an EntityResolver.
  InputStream in = new FileInputStream(inFile);
  closeables.add(in);
  InputSource fileSource = new InputSource();
  fileSource.setByteStream(in);
  fileSource.setSystemId(inFile.toURI().toString());

  SAXSource source = new SAXSource();
  XMLReader reader = XMLReaderFactory.createXMLReader();
  reader.setEntityResolver(new EntityResolver() {
    public InputSource resolveEntity(String publicId, String systemId)
        throws SAXException, IOException {
      if (systemId != null && systemId.endsWith("/djnml-1.0b.dtd")) {
        InputStream dtdIn = new FileInputStream(dtdFile);
        closeables.add(dtdIn);

        InputSource inputSource = new InputSource();
        inputSource.setByteStream(dtdIn);
        inputSource.setEncoding("UTF-8");

        return inputSource;
      }
      return null;
    }
  });

  source.setXMLReader(reader);
  source.setInputSource(fileSource);

  // Now we need to create a StreamResult.
  OutputStream out = new FileOutputStream(outputFile);
  closeables.add(out);
  StreamResult result = new StreamResult();
  result.setOutputStream(out);
  result.setSystemId(outputFile);

  // Create a templates object for the identity transform.  If you are going
  // to transform a lot of documents, you should do this once and
  // reuse the Templates object.
  InputStream identityIn = new FileInputStream(identityFile);
  closeables.add(identityIn);
  StreamSource identitySource = new StreamSource();
  identitySource.setSystemId(identityFile);
  identitySource.setInputStream(identityIn);
  TransformerFactory factory = TransformerFactory.newInstance();
  Templates templates = factory.newTemplates(identitySource);

  // Finally we need to create the transformer and do the transformation.
  Transformer transformer = templates.newTransformer();
  transformer.transform(source, result);

} finally {
  // Some older XML processors are bad at cleaning up input and output streams,
  // so we will do this manually.
  for (Closeable closeable : closeables) {
    if (closeable != null) {
      try {
        closeable.close();
      } catch (Exception e) {
      }
    }
  }
}
于 2012-11-16T10:07:04.683 回答