0

背景

我正在尝试以这样一种方式编写 Kryo 反序列化:如果一个对象数组包含一些无法反序列化的对象(由于代码更改),那么数组中的这些引用将变为 null 而不是抛出异常;允许回收对象的其余部分。我以前一直在使用 Java 的内置序列化,并且在其中我已经能够通过在数组中的每个项目之间写入一个“已知良好”的整数来实现这一点,然后在发生错误时在流中查找它以找到开始下一个对象。这详细介绍了反序列化包含一些不可反序列化对象的数组(挽救可反序列化部分)的问题。

出于效率的原因,我现在已经转移到 Kryo 序列化并尝试重新创建这种方法,但是在 Kryo 中,这种错误恢复似乎可以工作一次,但之后它就不能正确恢复。

我试过的

在序列化过程中,我试图END_OF_APPLE_MAGIC在 s 数组中的每个对象之间写入一个已知良好的整数() 。Apple在反序列化过程中,当BadApple找到无法反序列化的 a 时,将其替换为 a ErrorApple(类比越来越弱),并END_OF_APPLE_MAGIC搜索 a 以查找下一个苹果的位置。如果BadApple数组中有一个并且BadApple不是第一个条目,则此方法有效。但是如果数组中有超过 1 个BadApple或第一个AppleBadApple

public class AppleHolder implements Serializable,KryoSerializable{
    static int END_OF_APPLE_MAGIC=1234467895; //if this just "turns up" in the stream we will have a bad day; however this is only the case in recovery mode, so is an acceptable risk

    int numberOfApples=6;
    Apple[] apples=new Apple[numberOfApples];
    double otherData=15;

    //these are just for debug
    int dividers=0; //counts number of times END_OF_APPLE_MAGIC is found
    int problems=0; //counts number of times an apple fails to load
    int badIntegers=0; //counts number of times END_OF_APPLE_MAGIC is looked for and a different Integer is found (I have never seen this happen)

    public AppleHolder(){
        Apple goodApple=new Apple("GoodApple","tastyGood");
        BadApple badApple=new BadApple("BadApple","untastyBad");

        apples[0]=goodApple;
        apples[1]=badApple;
        apples[2]=goodApple;
        apples[3]=goodApple; // multiple references to same object intentional
        apples[4]=goodApple;
        apples[5]=goodApple;


    }


    public void write (Kryo kryo, Output output) {
        for(int i=0;i<apples.length;i++){

            //kryo.writeObject(output, apples[i]);
            kryo.writeClassAndObject(output, apples[i]);
            kryo.writeClassAndObject(output, END_OF_APPLE_MAGIC);
        }

        kryo.writeObject(output,otherData);
    }


    public void read (Kryo kryo, Input input) {
        try{

            apples =new Apple[numberOfApples];
            for(int i=0;i<apples.length;i++){

                try{
                    Object ob=kryo.readClassAndObject(input);
                    apples[i]=(Apple)ob;
                }catch(Exception e){
                    apples[i]=new ErrorApple();
                    problems++;
                }

                //Search for next Apple Boundary (except in recovery mode 
                //it will be the next entry)
                boolean atBoundary=false;
                while (atBoundary==false){ //should probably put a limit on this just in case
                    try{
                        int appleMagic =(Integer)kryo.readClassAndObject(input);
                        if (appleMagic == END_OF_APPLE_MAGIC){
                            atBoundary=true;
                            dividers++;
                        }else{
                            badIntegers++;
                        }
                    }catch(Exception e){
                        //painful byte reading mode only entered in recovery mode; under good 
                        //situations it does not represent an efficiency problem
                        input.skip(1); //consume byte of bad input
                        //Where buffer underflow exceptions occur they occur here
                    }
                }


            }
            otherData = kryo.readObject(input, Double.class);

        }catch(Exception e){
            //something when wrong (always a Buffer underflow so far), print what we have


            for(int i=0;i<apples.length;i++){
                System.out.println(apples[i]);

            }

            throw e;
        }

    }


    public static void main(String[] args)
            throws Exception {

        /*
         * (1) First run serialize()
         * (2) Rename/delete badApple such that it cannot be found for deserialization
         * (3) Run deSerialize(()
         */


        serialize();


        //deSerialize();

    }

    public static void serialize() throws Exception{
        AppleHolder testWrite = new AppleHolder();
        /*FileOutputStream fos = new FileOutputStream("testfile");
        ObjectOutputStream oos = new ObjectOutputStream(fos);
        oos.writeObject(testWrite);
        oos.flush();
        oos.close();
        */

        Kryo kryo = new Kryo();
        Output output = new Output(new FileOutputStream("testfile"));
        kryo.writeObject(output, testWrite);
        output.close();

    }

    public static void deSerialize() throws Exception{
        /*AppleHolder testRead;
        FileInputStream fis = new FileInputStream("testfile");
        ObjectInputStream ois = new ObjectInputStream(fis);
        testRead = (AppleHolder) ois.readObject();
        ois.close();
        */

        Kryo kryo = new Kryo();
        Input input = new Input(new FileInputStream("testfile"));
        AppleHolder testRead = kryo.readObject(input, AppleHolder.class);
        input.close();

        for(int i=0;i<testRead.apples.length;i++){
            System.out.println(testRead.apples[i]);

        }

        System.out.println("otherData: " + testRead.otherData);

    }

}

public class Apple implements Serializable {
    private String propertyOne;
    private String propertyTwo;

    public Apple(){}

    public Apple(String propertyOne, String propertyTwo) {
        this.propertyOne = propertyOne;
        this.propertyTwo = propertyTwo;
        validate();
    }

    private void writeObject(ObjectOutputStream o)
            throws IOException {

        o.writeObject(propertyOne);
        o.writeObject(propertyTwo);
    }

    private void readObject(ObjectInputStream o)
            throws IOException, ClassNotFoundException {

        propertyOne = (String) o.readObject();
        propertyTwo = (String) o.readObject();
        validate();
    }

    private void validate(){
        if(propertyOne == null ||
                propertyOne.length() == 0 ||
                propertyTwo == null ||
                propertyTwo.length() == 0){

            throw new IllegalArgumentException();
        }
    }

    public String getPropertyOne() {
        return propertyOne;
    }

    public String getPropertyTwo() {
        return propertyTwo;
    }

    @Override
    public String toString() {
        return "goodApple";
    }



}

public class BadApple extends Apple {


    public BadApple(){}

    public BadApple(String propertyOne, String propertyTwo) {
        super(propertyOne, propertyTwo);
    }

    @Override
    public String toString() {
        return "badApple";
    }
}

public class ErrorApple extends Apple {


    public ErrorApple(){}

    public ErrorApple(String propertyOne, String propertyTwo) {
        super(propertyOne, propertyTwo);
    }

    @Override
    public String toString() {
        return "errorApple";
    }

}

问题

如何挽救其中只有一些对象可反序列化的 Kyro 序列化数组?ErrorApple从而获得一个包含不可反序列化部分的条目的数组。在我的数组中,我在单个数组中有多个对同一对象的引用,必须在反序列化过程中保留这些引用。

所以进入序列化我有

[GoodApple]
[GoodApple]  
[GoodApple]  
[BadApple]
[BadApple]
[GoodApple] 

我想要从反序列化中出来(因为 badApple 已经改变并且不能反序列化

[GoodApple]
[GoodApple]  
[GoodApple]  
[ErrorApple]
[ErrorApple]
[GoodApple]  

我希望这可以提供一种后备方案,在这种情况下无法实现向后兼容性,或者删除了对我之前安装的程序的第 3 方修改

详细分析

本节概述了现有程序失败的方式。

一般来说

  • BadApple数组中第一个位置以外的单个位置将正常运行
  • 数组中第一个位置的ABadApple将导致下一次Apple正确读取,然后ErrorApples从那时起(即使是好的Apples)
  • 在超过 1个的情况下,第二个之后的BadApple第一个 good将正确读取,但可能会在数组中向前移动,然后从那时起(即使是 good s)。数组末尾会有一个并且可能有空条目。AppleBadAppleErrorApplesAppleKryoException: Buffer underflow

我使用的输入和输出如下所示:

    进出
    [好苹果] [好苹果]  
    [好苹果] [好苹果]  
    [坏苹果] [坏苹果]  
    [好苹果] [好苹果]  
    [好苹果] [好苹果]  
    [好苹果] [好苹果]  

    进出
    [坏苹果] [错误苹果]  
    [好苹果] [好苹果]  
    [goodApple] [errorApple]  
    [goodApple] [errorApple]  
    [goodApple] [errorApple]  
    [goodApple] [errorApple]  

    进出
    [好苹果] [好苹果]  
    [坏苹果] [错误苹果]  
    [好苹果] [好苹果]  
    [坏苹果] [错误苹果]  
    [好苹果] [好苹果]  
    [goodApple] [errorApple]  
    KryoException:缓冲区下溢。(发生在 input.skip(1);)  

    进出
    [好苹果] [好苹果]  
    [好苹果] [好苹果]  
    [坏苹果] [错误苹果]  
    [坏苹果] [错误苹果]  
    [好苹果] [好苹果]  
    [goodApple] [errorApple]  
    KryoException:缓冲区下溢(发生在 input.skip(1);)  

    进出
    [好苹果] [好苹果]  
    [坏苹果] [错误苹果]  
    [坏苹果] [错误苹果]  
    [坏苹果] [好苹果]  
    [goodApple] [errorApple]  
    [goodApple] [空]  
    KryoException:缓冲区下溢。(发生在 input.skip(1);)  

4

1 回答 1

1

我发现的一个解决方案是编写第二个恢复序列化文件,该文件在添加每个项目后只保存文件长度。然后,如果反序列化数据出现问题,程序就会知道在哪里可以找到“下一个好”的值。

但是有一个问题,一旦你读取了一个字节,你就无法重新读取它而不重新开始(.reset()抛出一个UnsupportedOperationException),有时 Kryo 会在它意识到它被一个坏对象阻塞之前开始读取下一个好的对象。我对此的解决方案是使用单独文件中的数据来检测要为每个对象读取多少字节,将它们作为字节读取,然后将其传递给 Kryo 进行反序列化。

所有这些都有一些开销,但对于我的测试,整个事情仍然比标准 Java 反序列化快得多,而且它只能在“恢复模式”下使用。

下面的程序证明了这一点,它将坏值保留为空值(但可以做任何它想做的事情)。

public class AppleHolder implements Serializable,KryoSerializable{
    int numberOfApples=10;
    Apple[] apples=new Apple[numberOfApples];
    double otherData=15;


    public AppleHolder(){

        BadApple sharedBad=new BadApple(0);

        for(int i=0;i<numberOfApples;i++){
            if (i>3 && i<=6){
                apples[i]=new Apple(i);
            }else{
                apples[i]=new BadApple();
            }
        }

    }


    public void write (Kryo kryo, Output output) {
        int[] recoveryTrack=new int[apples.length+1]; //last one for after the last entry

        for(int i=0;i<apples.length;i++){
            recoveryTrack[i]=output.total();
            kryo.writeClassAndObject(output, apples[i]);
        }
        recoveryTrack[recoveryTrack.length-1]=output.total();

        kryo.writeObject(output,otherData);

        Output outputRecovery;
        try {
            outputRecovery = new Output(new FileOutputStream("testfile.recovery"));
            kryo.writeObject(outputRecovery, recoveryTrack);
            outputRecovery.close();
        } catch (FileNotFoundException ex) {
            //I guess hopefully we won't need the recovery track
            Logger.getLogger(AppleHolder.class.getName()).log(Level.SEVERE, null, ex);
        }


    }


    public void read (Kryo kryo, Input input) {



        int[] readRecoveryTrack=null;
        try {
            Kryo kryoRecovery = new Kryo();
            Input inputRecovery = new Input(new FileInputStream("testfile.recovery"));
            readRecoveryTrack =kryoRecovery.readObject(inputRecovery, int[].class);
            inputRecovery.close();
        } catch (FileNotFoundException ex) {
            Logger.getLogger(AppleHolder.class.getName()).log(Level.SEVERE, null, ex);
        }





        apples=new Apple[numberOfApples];

        for(int j=0;j<apples.length;j++){

            int actualPos=input.total();
            int desiredPos=readRecoveryTrack[j];
            int desiredBytes=readRecoveryTrack[j+1]-readRecoveryTrack[j];

            byte[] bytes=input.readBytes(desiredBytes);

            ByteArrayInputStream byteStream =new  ByteArrayInputStream(bytes);

            try{
                apples[j]=(Apple)kryo.readClassAndObject(new Input(byteStream));
            }catch(Exception e){
                //don't care, leave null
            }




        }

    }

    public static void main(String[] args)
            throws Exception {

        /*
         * (1) First run serialize()
         * (2) Rename/delete badApple such that it cannot be found for deserialization
         * (3) Run deSerialize(()
         */


        serialize();


        //deSerialize();

    }

    public static void serialize() throws Exception{
        AppleHolder testWrite = new AppleHolder();


        Kryo kryo = new Kryo();
        Output output = new Output(new FileOutputStream("testfile"));
        kryo.writeObject(output, testWrite);
        output.close();
    }

    public static void deSerialize() throws Exception{

        Kryo kryo = new Kryo();
        Input input = new Input(new FileInputStream("testfile"));
        AppleHolder testRead = kryo.readObject(input, AppleHolder.class);
        input.close();


        for(int i=0;i<testRead.apples.length;i++){
            System.out.println(testRead.apples[i]);

        }

        System.out.println(testRead.otherData);

    }
}

public class Apple implements Serializable {
    protected int index;

    public Apple(){}

    public Apple(int index) {
        this.index = index;
    }


    @Override
    public String toString() {
        return "goodApple " + index;
    }



}

public class BadApple extends Apple {

    private static final long serialVersionUID = 7;

    public BadApple(){}

    public BadApple(int index){
        super(index);
    }


    @Override
    public String toString() {
        return "badApple " + index;
    }
}
于 2014-01-06T16:49:29.083 回答