我正在将大量数据加载到我的应用程序中。我有 40 多个模型，其中包含要加载的数据。除了这些表中的一个之外，所有表都可以正常导入。在一张表中，我遇到的每一行都重复的问题，所以我最终得到了 468,450 行，而不是 234,225 行。绝对不是我要找的。每一行都是完美复制的。
作为此导入的一部分，我还将扁平化数据结构。所以我将查找字段填充到主 sku 表中。
这是我用来导入此 skus 表的代码。
命名空间：upload_data 做
desc "Load Skus"
task :load_skus => :environment do
require 'csv'
require 'net/ftp'
start_time = Time.now
ftp = Net::FTP.new('<ftp location>')
ftp.login(user = "<username>", passwd = "<password>")
ftp.passive = true
ftp.get("sku.csv", File.basename( "/data/sku.csv"))
ftp.quit()
upload = File.join('sku.csv')
logcount=0
CSV.foreach(upload, :headers => false) do |row|
# roa = Roa.find_by_roa_code(row[21])
# if roa
# roa_name = roa.roa_name
# else
# roa_name = nil
# end
#ROA
roa = Roa.find_by_roa_code(row[21])
roa_name = roa && roa.roa_name
#THER
ther = Ther.find_by_thera_class_code_int(row[50] && row[50].to_i)
thera_class_desc = ther && ther.thera_class_desc
#DEA
dea = Dea.find_by_dea_class_code(row[45])
dea_class_desc = dea && dea.dea_class_desc
#LBLY2K
lbly2k = Lbly2k.find_by_mfg_code(row[56])
labeler_code = lbly2k && lbly2k.labeler_code
rebate_status = lbly2k && lbly2k.rebate_status
rebate_effective_date = lbly2k && lbly2k.rebate_effective_date
rebate_termination_date = lbly2k && lbly2k.rebate_termination_date
#FORM
form = Form.find_by_master_form_code(row[22])
form_desc = form && form.form_desc
#PRODCAT
prodcat = Prodcat.find_by_product_category(row[26] && row[26].to_i.to_s)
product_cat_desc = prodcat && prodcat.product_cat_desc
#EXPGENRC
expgenrc = Expgenrc.find_by_gcr_code(row[52])
gcr_exp_name = expgenrc && expgenrc.gcr_exp_name
#GCR
gcr = Gcr.find_by_gcr_code(row[52])
gcr_name = gcr && gcr.gcr_name
#MFORM
mform = Mform.find_by_master_form_code(row[22])
master_form_desc = mform && mform.master_form_desc
#Orange Book Data
orange = Orange.find_by_orange_book_code(row[31])
orange_book_desc = orange && orange.orange_book_desc
#EXDRUG
exdrug = Exdrug.find_by_exceptional_drug(row[44])
exceptional_drug_desc = exdrug && exdrug.exceptional_drug_desc
#MEASURE
measure = Measure.find_by_measure_code(row[39])
measure_desc = measure && measure.measure_desc
#JCODE
jcode = Jcode.find_by_ndc(row[13] && row[13].gsub('-', ''))
puts "found jcode: #{jcode.inspect}"
j_code = jcode && jcode.j_code
j_code_desc = jcode && jcode.j_code_desc
#GFC
gfc = Gfc.find_by_gcr_code(row[52])
gfc_master_code = gfc && gfc.gfc_master_code
gfc_deact = gfc && gfc.gfc_deact
#packstren
packstren = Packstren.find_by_ndc(row[13] && row[13].gsub('-', ''))
puts "Found Packstren #{packstren.inspect}"
package_ID = packstren && packstren.package_ID
puts "Package_ID is #{package_ID}"
strength_code = packstren && packstren.strength_code
#STRENGTH
strength = Strength.find_by_strength_code(strength_code)
puts "Found strength: #{strength.inspect}"
strength_name_long = strength && strength.strength_name_long
puts "Strength name long is #{strength_name_long}"
#EXPSTREN
expstren = Expstren.find_by_strength_code(strength_code)
strength_exp_name = expstren && expstren.strength_exp_name
puts "Strength_exp_name is #{strength_exp_name}"
#AWP
awp = Awp.find_by_ndc(row[13] && row[13].gsub('-', ''))
puts "Found awp: #{awp.inspect}"
effective_date = awp && awp.effective_date
puts "found effective_date: #{effective_date}"
manufacturer_cmsid = awp && awp.manufacturer_cmsid
puts "found manufacturer_cmsid: #{manufacturer_cmsid}"
package_cmsid = awp && awp.package_cmsid
puts "found package_cmsid: #{package_cmsid}"
Sku.create(:record_change => row[0],
:record_change_date => row[1],
:price_change_date => row[2],
:add_flag => row[3],
:add_date => row[4],
:deactivate_flag => row[5],
:deactivate_date => row[6],
:reactivate_flag => row[7],
:reactivate_date => row[8],
:filler1 => row[9],
:prev_ndc_awp_eff_date => row[10],
:prev_ndc_awp_pack_price => row[11],
:prev_ndc_disc_date => row[12],
:ndc => row[13].gsub('-',''),
:j_code => j_code,
:j_code_desc => j_code_desc,
:package_ID => package_ID,
:strength_code => strength_code,
:filler2 => row[14],
:ndc_orig_config => row[15].gsub('-',''),
:ndc10 => row[16].gsub('-',''),
:ndc_change_flag => row[17],
:prev_ndc => row[18],
:prev_ndc_config => row[19],
:prev_ndc10 => row[20],
:roa_code => row[21],
:roa_name => roa_name,
# :roa_name => Roa.find_by_roa_code(row[21]) && Roa.find_by_roa_code(row[21]).roa_name,
:master_form_code => row[22],
:master_form_desc => master_form_desc,
:form_desc => form_desc,
:product_name => row[23],
:additional_desc => row[24],
:mfg_name_long => row[25],
:product_category => row[26],
:product_cat_desc => product_cat_desc,
:solid_liquid => row[27],
:form_code => row[28],
:metric_size => row[29],
:strength_name => row[30],
:orange_book_code => row[31],
:orange_book_desc => orange_book_desc,
:filler3 => row[32],
:orange_book_std_flag => row[33],
:unit_dose_flag => row[34],
:dispensing_unit_flag => row[35],
:package_size => row[36],
:package_qty_code => row[37],
:product_size => row[38],
:measure_code => row[39],
:measure_desc => measure_desc,
:filler4 => row[40],
:top_volume_rank => row[41],
:single_source_flag => row[42],
:maintenance_drug_flag => row[43],
:exceptional_drug => row[44],
:exceptional_drug_desc => exceptional_drug_desc,
:dea_class_code => row[45],
:dea_class_desc => dea_class_desc,
:filler5 => row[46],
:filler6 => row[47],
:desi_drug_flag => row[48],
:desi_effective_date => row[49],
:thera_class_code => row[50],
:thera_class_desc => thera_class_desc,
:generic_class_code => row[51],
:gcr_code => row[52],
:gcr_exp_name => gcr_exp_name,
:gcr_name => gcr_name,
:gfc_master_code => gfc_master_code,
:gfc_deact => gfc_deact,
:gfc_code => row[53],
:stc_package_size => row[54],
:unit_of_measure => row[55],
:mfg_code => row[56],
:labeler_code => labeler_code,
:rebate_status => rebate_status,
:rebate_effective_date => rebate_effective_date,
:rebate_termination_date => rebate_termination_date,
:wac_start_flag => row[57],
:wac_p_price => row[58],
:wac_u_price => row[59],
:wac_eff_date => row[60],
:wac1pre_p_price => row[61],
:wac1pre_u_price => row[62],
:wac1pre_eff_date => row[63],
:wac2pre_p_price => row[64],
:wac2pre_u_price => row[65],
:wac2pre_eff_date => row[66],
:filler7 => row[67],
:filler8 => row[68],
:filler9 => row[69],
:awp_start_flag => row[70],
:awp_p_price => row[71],
:awp_u_price => row[72],
:awp_eff_date => row[73],
:awp1pre_p_price => row[74],
:awp1pre_u_price => row[75],
:awp1pre_eff_date => row[76],
:awp2pre_p_price => row[77],
:awp2pre_u_price => row[78],
:awp2pre_eff_date => row[79],
:dp_start_flag => row[80],
:dp_p_price => row[81],
:dp_u_price => row[82],
:dp_eff_date => row[83],
:dp1pre_p_price => row[84],
:dp1pre_u_price => row[85],
:dp1pre_eff_date => row[86],
:dp2pre_p_price => row[87],
:dp2pre_u_price => row[88],
:dp2pre_eff_date => row[89],
:ful_start_flag => row[90],
:ful_p_price => row[91],
:ful_u_price => row[92],
:ful_eff_date => row[93],
:ful_pre_p_price => row[94],
:ful_pre_u_price => row[95],
:ful_pre_eff_date => row[96],
:ful2_pre_p_price => row[97],
:ful2_pre_u_price => row[98],
:ful2_pre_eff_date => row[99],
:srp_start_flag => row[100],
:srp_p_price => row[102],
:srp_u_price => row[102],
:srp_eff_date => row[103],
:srp1pre_p_price => row[104],
:srp1pre_u_price => row[105],
:srp1pre_eff_date => row[106],
:srp2pre_p_price => row[107],
:srp2pre_u_price => row[108],
:srp2pre_eff_date => row[109])
logcount += 1
end
total_time = Time.now - start_time
puts "Successfully added #{logcount} Skus. Elapsed time #{total_time} seconds."
end
end
我正在做什么导致每一行被导入的任何想法？
谢谢