node.js - 关于从 CSV 文件中获取的 NodeJS 后端重复数据

Question

我在 Node JS 中创建了一个后端来从 CSV 文件中获取数据。

const multer = require('multer');
const csv = require('fast-csv');
const mongodb = require('mongodb');
const fs = require('fs');
const express = require('express');
const app = express();

// Allow CORS
app.use(function(req, res, next) {
    res.header("Access-Control-Allow-Origin", "http://localhost:3000");
    res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
    next();
});

// Set Global Directory
global.__basedir = __dirname;
// Multer Storage
const storage = multer.diskStorage({
    destination: (req, file, cb) => {
        cb(null, __basedir + '/uploads/')
    },
    filename: (req, file, cb) => {
        cb(null, file.fieldname + "-" + Date.now() + "-" + file.originalname)
    }
})

const csvFilter = (req, file, cb) => {
    if(file.mimetype.includes("csv")) {
        cb(null, true)
    } else {
        cb("Please upload only csv file", false)
    }
}

const upload = multer({ storage: storage, fileFilter: csvFilter })

// Upload csv
app.post('/api/upload-csv-file', upload.single("file"), (req, res) => {
    try {
        if(req.file == undefined) {
            return res.status(400).send({
                message: "Please upload a csv file!"
            })
        }
        
        let csvData= []
        let path = __basedir + '/uploads/' + req.file.filename
        fs.createReadStream(path)
            .pipe(csv.parse({headers: true}))
            .on("error", (error) => {
                throw error.message
            })
            .on("data", (row) => {
                csvData.push(row)
            })
            .on("end", () => {
                var url = "mongodb://localhost:27017/TestDB";
                var dbConn;
                mongodb.MongoClient.connect(url, {
                    useUnifiedTopology: true
                }).then((client) => {
                    console.log('DB Connected!')
                    dbConn = client.db()

                    var collectionName = 'statistics'
                    var collection = dbConn.collection(collectionName)
                    collection.insertMany(csvData, (err, result) => {
                        if(err) console.log(err)
                        if(result) {
                            res.status(200).send({
                                message: "Upload/import the CSV data into database successfully: " 
                                + req.file.originalname
                            })
                        }
                    })
                }).catch(err => {
                    res.status(500).send({
                        message: "Fail to import data into database!",
                        error: err.message,
                    })
                })
            })
    } catch(error) {
        console.log("catch error~", error)
        res.status(500).send({
            message: "Could not upload the file: " + req.file.originalname
        })
    }
})

app.get('/api/statistics', (req,res) => {
    var url = "mongodb://localhost:27017/TestDB";
    var dbConn;
    mongodb.MongoClient.connect(url, {
        useUnifiedTopology: true,
    }).then((client) => {
        dbConn = client.db()

        var collectionName = 'statistics'
        var collection = dbConn.collection(collectionName)
        collection.find().toArray(function(err, result){
            if(err) throw err
            res.status(200).send({ statistics : result })
            client.close()
        })
    }).catch(err => {
        res.status(500).send({
            message: "Fail to fetch data from database!",
            error: err.message,
        })
    })
})

// Create server
let server =  app.listen(5000, function() {
    let port = server.address().port
    console.log("App Server running at - https://localhost:%s", port)
})

然后我从 CSV 文件上传了以下数据：

Time,Value
"2020-11-03T01:00:00+01:00",0
"2020-11-04T01:00:00+01:00",0
"2020-11-05T01:00:00+01:00",0
"2020-11-06T01:00:00+01:00",0
"2020-11-07T01:00:00+01:00",0
"2020-11-08T01:00:00+01:00",0
"2020-11-09T01:00:00+01:00",0
"2020-11-10T01:00:00+01:00",0
"2020-11-11T01:00:00+01:00",0
"2020-11-12T01:00:00+01:00",0
"2020-11-13T01:00:00+01:00",0
"2020-11-14T01:00:00+01:00",0
"2020-11-15T01:00:00+01:00",0
"2020-11-16T01:00:00+01:00",0
"2020-11-17T01:00:00+01:00",0
"2020-11-18T01:00:00+01:00",0
"2020-11-19T01:00:00+01:00",0
"2020-11-20T01:00:00+01:00",0
"2020-11-21T01:00:00+01:00",0
"2020-11-22T01:00:00+01:00",0
"2020-11-23T01:00:00+01:00",0
"2020-11-24T01:00:00+01:00",0
"2020-11-25T01:00:00+01:00",0
"2020-11-26T01:00:00+01:00",0
"2020-11-27T01:00:00+01:00",0
"2020-11-28T01:00:00+01:00",0
"2020-11-29T01:00:00+01:00",0
"2020-11-30T01:00:00+01:00",0
"2020-12-01T01:00:00+01:00",0
"2020-12-02T01:00:00+01:00",0
"2020-12-03T01:00:00+01:00",0
"2020-12-04T01:00:00+01:00",0
"2020-12-05T01:00:00+01:00",0
"2020-12-06T01:00:00+01:00",0
"2020-12-07T01:00:00+01:00",0
"2020-12-08T01:00:00+01:00",0
"2020-12-09T01:00:00+01:00",0
"2020-12-10T01:00:00+01:00",0
"2020-12-11T01:00:00+01:00",0
"2020-12-12T01:00:00+01:00",0
"2020-12-13T01:00:00+01:00",0
"2020-12-14T01:00:00+01:00",0
"2020-12-15T01:00:00+01:00",0
"2020-12-16T01:00:00+01:00",0
"2020-12-17T01:00:00+01:00",0
"2020-12-18T01:00:00+01:00",0
"2020-12-19T01:00:00+01:00",0
"2020-12-20T01:00:00+01:00",0
"2020-12-21T01:00:00+01:00",0
"2020-12-22T01:00:00+01:00",0
"2020-12-23T01:00:00+01:00",0
"2020-12-24T01:00:00+01:00",0
"2020-12-25T01:00:00+01:00",0
"2020-12-26T01:00:00+01:00",0
"2020-12-27T01:00:00+01:00",0
"2020-12-28T01:00:00+01:00",0
"2020-12-29T01:00:00+01:00",0
"2020-12-30T01:00:00+01:00",0
"2020-12-31T01:00:00+01:00",0
"2021-01-01T01:00:00+01:00",0
"2021-01-02T01:00:00+01:00",0
"2021-01-03T01:00:00+01:00",0
"2021-01-04T01:00:00+01:00",0
"2021-01-05T01:00:00+01:00",0
"2021-01-06T01:00:00+01:00",0
"2021-01-07T01:00:00+01:00",0
"2021-01-08T01:00:00+01:00",0
"2021-01-09T01:00:00+01:00",0
"2021-01-10T01:00:00+01:00",0
"2021-01-11T01:00:00+01:00",0
"2021-01-12T01:00:00+01:00",0
"2021-01-13T01:00:00+01:00",0
"2021-01-14T01:00:00+01:00",0
"2021-01-15T01:00:00+01:00",0
"2021-01-16T01:00:00+01:00",0
"2021-01-17T01:00:00+01:00",0
"2021-01-18T01:00:00+01:00",0
"2021-01-19T01:00:00+01:00",0
"2021-01-20T01:00:00+01:00",0
"2021-01-21T01:00:00+01:00",0
"2021-01-22T01:00:00+01:00",0
"2021-01-23T01:00:00+01:00",0
"2021-01-24T01:00:00+01:00",0
"2021-01-25T01:00:00+01:00",0
"2021-01-26T01:00:00+01:00",0
"2021-01-27T01:00:00+01:00",0
"2021-01-28T01:00:00+01:00",0
"2021-01-29T01:00:00+01:00",0
"2021-01-30T01:00:00+01:00",0
"2021-01-31T01:00:00+01:00",0
"2021-02-01T01:00:00+01:00",0
"2021-02-02T01:00:00+01:00",0
"2021-02-03T01:00:00+01:00",0
"2021-02-04T01:00:00+01:00",0
"2021-02-05T01:00:00+01:00",0
"2021-02-06T01:00:00+01:00",0
"2021-02-07T01:00:00+01:00",0
"2021-02-08T01:00:00+01:00",0
"2021-02-09T01:00:00+01:00",0
"2021-02-10T01:00:00+01:00",0
"2021-02-11T01:00:00+01:00",0
"2021-02-12T01:00:00+01:00",0
"2021-02-13T01:00:00+01:00",0
"2021-02-14T01:00:00+01:00",0
"2021-02-15T01:00:00+01:00",0
"2021-02-16T01:00:00+01:00",0
"2021-02-17T01:00:00+01:00",0
"2021-02-18T01:00:00+01:00",0
"2021-02-19T01:00:00+01:00",0
"2021-02-20T01:00:00+01:00",0
"2021-02-21T01:00:00+01:00",0
"2021-02-22T01:00:00+01:00",0
"2021-02-23T01:00:00+01:00",0
"2021-02-24T01:00:00+01:00",0
"2021-02-25T01:00:00+01:00",0
"2021-02-26T01:00:00+01:00",0
"2021-02-27T01:00:00+01:00",0
"2021-02-28T01:00:00+01:00",0
"2021-03-01T01:00:00+01:00",0
"2021-03-02T01:00:00+01:00",0
"2021-03-03T01:00:00+01:00",0
"2021-03-04T01:00:00+01:00",0
"2021-03-05T01:00:00+01:00",0
"2021-03-06T01:00:00+01:00",0
"2021-03-07T01:00:00+01:00",0
"2021-03-08T01:00:00+01:00",0
"2021-03-09T01:00:00+01:00",0
"2021-03-10T01:00:00+01:00",0
"2021-03-11T01:00:00+01:00",0
"2021-03-12T01:00:00+01:00",0
"2021-03-13T01:00:00+01:00",0
"2021-03-14T01:00:00+01:00",0
"2021-03-15T01:00:00+01:00",0
"2021-03-16T01:00:00+01:00",0
"2021-03-17T01:00:00+01:00",0
"2021-03-18T01:00:00+01:00",0
"2021-03-19T01:00:00+01:00",0
"2021-03-20T01:00:00+01:00",0
"2021-03-21T01:00:00+01:00",0
"2021-03-22T01:00:00+01:00",0
"2021-03-23T01:00:00+01:00",0
"2021-03-24T01:00:00+01:00",0
"2021-03-25T01:00:00+01:00",0
"2021-03-26T01:00:00+01:00",0
"2021-03-27T01:00:00+01:00",0
"2021-03-28T01:00:00+01:00",0
"2021-03-29T02:00:00+02:00",0
"2021-03-30T02:00:00+02:00",0
"2021-03-31T02:00:00+02:00",0
"2021-04-01T02:00:00+02:00",0
"2021-04-02T02:00:00+02:00",0
"2021-04-03T02:00:00+02:00",0
"2021-04-04T02:00:00+02:00",0
"2021-04-05T02:00:00+02:00",0
"2021-04-06T02:00:00+02:00",0
"2021-04-07T02:00:00+02:00",0
"2021-04-08T02:00:00+02:00",0
"2021-04-09T02:00:00+02:00",0
"2021-04-10T02:00:00+02:00",0
"2021-04-11T02:00:00+02:00",0
"2021-04-12T02:00:00+02:00",0
"2021-04-13T02:00:00+02:00",0
"2021-04-14T02:00:00+02:00",0
"2021-04-15T02:00:00+02:00",0
"2021-04-16T02:00:00+02:00",0
"2021-04-17T02:00:00+02:00",0
"2021-04-18T02:00:00+02:00",0
"2021-04-19T02:00:00+02:00",0
"2021-04-20T02:00:00+02:00",0
"2021-04-21T02:00:00+02:00",0
"2021-04-22T02:00:00+02:00",0
"2021-04-23T02:00:00+02:00",0
"2021-04-24T02:00:00+02:00",0
"2021-04-25T02:00:00+02:00",0
"2021-04-26T02:00:00+02:00",0
"2021-04-27T02:00:00+02:00",0
"2021-04-28T02:00:00+02:00",0
"2021-04-29T02:00:00+02:00",0
"2021-04-30T02:00:00+02:00",0
"2021-05-01T02:00:00+02:00",0
"2021-05-02T02:00:00+02:00",0
"2021-05-03T02:00:00+02:00",0
"2021-05-04T02:00:00+02:00",0
"2021-05-05T02:00:00+02:00",0
"2021-05-06T02:00:00+02:00",0
"2021-05-07T02:00:00+02:00",0
"2021-05-08T02:00:00+02:00",0
"2021-05-09T02:00:00+02:00",0
"2021-05-10T02:00:00+02:00",0
"2021-05-11T02:00:00+02:00",0
"2021-05-12T02:00:00+02:00",0
"2021-05-13T02:00:00+02:00",0
"2021-05-14T02:00:00+02:00",0
"2021-05-15T02:00:00+02:00",0
"2021-05-16T02:00:00+02:00",0
"2021-05-17T02:00:00+02:00",0
"2021-05-18T02:00:00+02:00",0
"2021-05-19T02:00:00+02:00",0
"2021-05-20T02:00:00+02:00",0
"2021-05-21T02:00:00+02:00",0
"2021-05-22T02:00:00+02:00",0
"2021-05-23T02:00:00+02:00",0
"2021-05-24T02:00:00+02:00",0
"2021-05-25T02:00:00+02:00",0
"2021-05-26T02:00:00+02:00",0
"2021-05-27T02:00:00+02:00",0
"2021-05-28T02:00:00+02:00",0
"2021-05-29T02:00:00+02:00",0
"2021-05-30T02:00:00+02:00",0
"2021-05-31T02:00:00+02:00",0
"2021-06-01T02:00:00+02:00",0
"2021-06-02T02:00:00+02:00",0
"2021-06-03T02:00:00+02:00",0
"2021-06-04T02:00:00+02:00",0
"2021-06-05T02:00:00+02:00",0
"2021-06-06T02:00:00+02:00",0
"2021-06-07T02:00:00+02:00",0
"2021-06-08T02:00:00+02:00",0
"2021-06-09T02:00:00+02:00",0
"2021-06-10T02:00:00+02:00",0
"2021-06-11T02:00:00+02:00",0
"2021-06-12T02:00:00+02:00",0
"2021-06-13T02:00:00+02:00",0
"2021-06-14T02:00:00+02:00",0
"2021-06-15T02:00:00+02:00",0
"2021-06-16T02:00:00+02:00",0
"2021-06-17T02:00:00+02:00",0
"2021-06-18T02:00:00+02:00",0
"2021-06-19T02:00:00+02:00",0
"2021-06-20T02:00:00+02:00",0
"2021-06-21T02:00:00+02:00",0
"2021-06-22T02:00:00+02:00",0
"2021-06-23T02:00:00+02:00",7
"2021-06-24T02:00:00+02:00",13
"2021-06-25T02:00:00+02:00",31
"2021-06-26T02:00:00+02:00",42
"2021-06-27T02:00:00+02:00",0
"2021-06-28T02:00:00+02:00",494
"2021-06-29T02:00:00+02:00",697
"2021-06-30T02:00:00+02:00",705
"2021-07-01T02:00:00+02:00",834
"2021-07-02T02:00:00+02:00",757
"2021-07-03T02:00:00+02:00",0
"2021-07-04T02:00:00+02:00",0
"2021-07-05T02:00:00+02:00",632
"2021-07-06T02:00:00+02:00",438
"2021-07-07T02:00:00+02:00",745
"2021-07-08T02:00:00+02:00",608
"2021-07-09T02:00:00+02:00",914
"2021-07-10T02:00:00+02:00",0
"2021-07-11T02:00:00+02:00",0
"2021-07-12T02:00:00+02:00",611
"2021-07-13T02:00:00+02:00",624
"2021-07-14T02:00:00+02:00",984
"2021-07-15T02:00:00+02:00",665
"2021-07-16T02:00:00+02:00",610
"2021-07-17T02:00:00+02:00",0
"2021-07-18T02:00:00+02:00",0
"2021-07-19T02:00:00+02:00",618
"2021-07-20T02:00:00+02:00",631
"2021-07-21T02:00:00+02:00",0
"2021-07-22T02:00:00+02:00",481
"2021-07-23T02:00:00+02:00",615
"2021-07-24T02:00:00+02:00",0
"2021-07-25T02:00:00+02:00",0
"2021-07-26T02:00:00+02:00",872
"2021-07-27T02:00:00+02:00",749
"2021-07-28T02:00:00+02:00",1032
"2021-07-29T02:00:00+02:00",770
"2021-07-30T02:00:00+02:00",947
"2021-07-31T02:00:00+02:00",0
"2021-08-01T02:00:00+02:00",0
"2021-08-02T02:00:00+02:00",1570
"2021-08-03T02:00:00+02:00",1118
"2021-08-04T02:00:00+02:00",1508
"2021-08-05T02:00:00+02:00",1685
"2021-08-06T02:00:00+02:00",1187
"2021-08-07T02:00:00+02:00",1286
"2021-08-08T02:00:00+02:00",0
"2021-08-09T02:00:00+02:00",1204
"2021-08-10T02:00:00+02:00",831
"2021-08-11T02:00:00+02:00",836
"2021-08-12T02:00:00+02:00",774
"2021-08-13T02:00:00+02:00",829
"2021-08-14T02:00:00+02:00",880
"2021-08-15T02:00:00+02:00",0
"2021-08-16T02:00:00+02:00",1140
"2021-08-17T02:00:00+02:00",1238
"2021-08-18T02:00:00+02:00",1374
"2021-08-19T02:00:00+02:00",964
"2021-08-20T02:00:00+02:00",862
"2021-08-21T02:00:00+02:00",1670
"2021-08-22T02:00:00+02:00",0
"2021-08-23T02:00:00+02:00",1160
"2021-08-24T02:00:00+02:00",1033
"2021-08-25T02:00:00+02:00",685
"2021-08-26T02:00:00+02:00",1512
"2021-08-27T02:00:00+02:00",1453
"2021-08-28T02:00:00+02:00",2092
"2021-08-29T02:00:00+02:00",0
"2021-08-30T02:00:00+02:00",1429
"2021-08-31T02:00:00+02:00",1244
"2021-09-01T02:00:00+02:00",861
"2021-09-02T02:00:00+02:00",847
"2021-09-03T02:00:00+02:00",735
"2021-09-04T02:00:00+02:00",1186
"2021-09-05T02:00:00+02:00",0
"2021-09-06T02:00:00+02:00",686
"2021-09-07T02:00:00+02:00",676
"2021-09-08T02:00:00+02:00",711
"2021-09-09T02:00:00+02:00",650
"2021-09-10T02:00:00+02:00",763
"2021-09-11T02:00:00+02:00",1448
"2021-09-12T02:00:00+02:00",0
"2021-09-13T02:00:00+02:00",635
"2021-09-14T02:00:00+02:00",411
"2021-09-15T02:00:00+02:00",601
"2021-09-16T02:00:00+02:00",731
"2021-09-17T02:00:00+02:00",770
"2021-09-18T02:00:00+02:00",962
"2021-09-19T02:00:00+02:00",0
"2021-09-20T02:00:00+02:00",602
"2021-09-21T02:00:00+02:00",594
"2021-09-22T02:00:00+02:00",709
"2021-09-23T02:00:00+02:00",540
"2021-09-24T02:00:00+02:00",564
"2021-09-25T02:00:00+02:00",354
"2021-09-26T02:00:00+02:00",0
"2021-09-27T02:00:00+02:00",574
"2021-09-28T02:00:00+02:00",628
"2021-09-29T02:00:00+02:00",831
"2021-09-30T02:00:00+02:00",632
"2021-10-01T02:00:00+02:00",756
"2021-10-02T02:00:00+02:00",1003
"2021-10-03T02:00:00+02:00",0
"2021-10-04T02:00:00+02:00",505
"2021-10-05T02:00:00+02:00",754
"2021-10-06T02:00:00+02:00",803
"2021-10-07T02:00:00+02:00",825
"2021-10-08T02:00:00+02:00",829
"2021-10-09T02:00:00+02:00",1324
"2021-10-10T02:00:00+02:00",0
"2021-10-11T02:00:00+02:00",506
"2021-10-12T02:00:00+02:00",641
"2021-10-13T02:00:00+02:00",803
"2021-10-14T02:00:00+02:00",729
"2021-10-15T02:00:00+02:00",774
"2021-10-16T02:00:00+02:00",1191
"2021-10-17T02:00:00+02:00",0
"2021-10-18T02:00:00+02:00",655
"2021-10-19T02:00:00+02:00",571
"2021-10-20T02:00:00+02:00",576
"2021-10-21T02:00:00+02:00",606
"2021-10-22T02:00:00+02:00",676
"2021-10-23T02:00:00+02:00",1306
"2021-10-24T02:00:00+02:00",0
"2021-10-25T02:00:00+02:00",671
"2021-10-26T02:00:00+02:00",797
"2021-10-27T02:00:00+02:00",666
"2021-10-28T02:00:00+02:00",658
"2021-10-29T02:00:00+02:00",678
"2021-10-30T02:00:00+02:00",1211
"2021-10-31T02:00:00+02:00",0
"2021-11-01T01:00:00+01:00",0
"2021-11-02T01:00:00+01:00",952
"2021-11-03T01:00:00+01:00",641

我在邮递员中创建了一个get请求，然后我看到相同的数据不断重复多次，如下：邮递员的截图

952 在 CSV 文件中仅存在一次，而在邮递员获取请求窗口中显示 38 次。

我究竟做错了什么？

node.js - 关于从 CSV 文件中获取的 NodeJS 后端重复数据

0 回答 0

Related

Reference