0

BLOBS在过去的大约 5 年里，我们每天都在运行一个存储过程来从数据库中提取数据。我们通常每晚提取大约 25 个文件，其中大部分大小约为 500KB，少数接近 10,000KB。

这个过程从来都不是最快的，但在我们移动数据中心后，这个过程可能需要 12 个小时以上。当您仅提取约 55MB 左右时，这本身就令人震惊。我们已经聘请了所有相关团队来查看 Oracle 的性能、磁盘 I/O 等，他们声称一切都很完美。

我一直在阅读UTL_FILEand DBMS_LOB.read，看到人们谈论pos在每个循环之后重置等。老实说，我似乎无法弄清楚这意味着什么，并且普遍的共识是有更好的方法来实现同样的结果。

不幸的是，我们没有重构它的自由，所以任何人都可以看到我们的程序有什么明显的错误吗？我只是在为一些我不完全理解的事情而苦苦挣扎，而那些维护我们基础设施的人将这一切都归咎于这段代码并洗了他们的手。

    CREATE OR REPLACE PROCEDURE PKG_EXTRACT (l_brand IN VARCHAR2) AS

   l_file      UTL_FILE.FILE_TYPE;
   l_buffer    RAW(32767);
   l_amount    BINARY_INTEGER := 32767;
   l_pos       INTEGER;
   l_blob      BLOB;
   l_blob_len  INTEGER;
   x  NUMBER;
   l_file_name VARCHAR2(200);
   l_count       INTEGER := 1;
   v_code  NUMBER;
   v_errm  VARCHAR2(64);
 log_file      UTL_FILE.FILE_TYPE;
rec_num number;

BEGIN
DECLARE
   CURSOR extract_cur IS
      SELECT DATA, BIN_NAME
      FROM   STAGING
      WHERE UPPER(EXTRACTED)='N';

BEGIN
log_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR','pkg_extract.log','a', 32767);

UTL_FILE.put_line(log_file,'Logging is being done in 24 hours format - V1.5 ',TRUE);
 UTL_FILE.put_line(log_file,'Extract procedure started on Date-Time = '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE);


select count(1) into rec_num from staging;

UTL_FILE.put_line(log_file,'Total Number of records found = ' || rec_num , TRUE);
select count(1) into rec_num from staging where UPPER(EXTRACTED)='N';
UTL_FILE.put_line(log_file,'Total Number of records matching criteria = ' || rec_num , TRUE);

   dbms_output.put_line('Loop through records and write them to file');
   FOR extract_rec IN extract_cur
   LOOP

      l_pos := 1;
      l_blob := extract_rec.DATA;
      l_blob_len := DBMS_LOB.getlength(l_blob);

      -- Save blob length.
      x := l_blob_len;

      l_file_name := extract_rec.BIN_NAME ;

      -- Open the destination file.
      dbms_output.put_line('Open the destination file:- ' || l_file_name);
      l_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR',l_file_name,'wb', 32767);
      dbms_output.put_line('File opened');

      -- Read chunks of the BLOB and write them to the file until complete.
      dbms_output.put_line('l_pos:- ' || l_pos);
      dbms_output.put_line('l_blob_len:- ' || l_blob_len);
      WHILE l_pos <= l_blob_len
      LOOP
         dbms_output.put_line('DBMS_LOB.read from position: ' || l_pos);
         DBMS_LOB.read(l_blob, l_amount, l_pos, l_buffer);
         dbms_output.put_line('UTL_FILE.put_raw');
         UTL_FILE.put_raw(l_file, l_buffer, TRUE);
         dbms_output.put_line('Written ' || l_amount || ' bytes of data starting at position: ' || l_pos);

         -- Set the start position for the next cut.
         l_pos := l_pos + l_amount;

    --updating the extract field

        dbms_output.put_line(extract_rec.BIN_NAME);

      END LOOP;

      l_count := l_count + 1;
      -- Close the file.
      dbms_output.put_line('Close the file:- ' || l_file_name);
      UTL_FILE.fclose(l_file);

      update staging set extracted='Y', extract_timestamp=sysdate where bin_name=extract_rec.BIN_NAME;
      commit;

   END LOOP;

 UTL_FILE.put_line(log_file,'Extract procedure Completed on Date-Time = '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE);

 IF UTL_FILE.is_open(log_file) THEN
      UTL_FILE.fclose(log_file);
end if;
END;

EXCEPTION
   WHEN OTHERS THEN
   v_code := SQLCODE;
   v_errm := SUBSTR(SQLERRM, 1, 64);
   dbms_output.put_line('Error code ' || v_code || ': ' || v_errm);
UTL_FILE.put_line(log_file,'--------------------------------------' ,TRUE);
 UTL_FILE.put_line(log_file,'Error Occurred while executing '||'Error code ' || v_code || ': ' || v_errm ,TRUE);
 UTL_FILE.put_line(log_file,'Extract procedure Completed with errors - '|| TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),TRUE);
UTL_FILE.put_line(log_file,'--------------------------------------' ,TRUE);

   -- Close the file if something goes wrong.
   IF UTL_FILE.is_open(l_file) THEN

      UTL_FILE.fclose(l_file);

 IF UTL_FILE.is_open(log_file) THEN
      UTL_FILE.fclose(log_file);
end if;
   END IF;
   RAISE;
END;
/

编辑

的执行计划CURSOR extract_cur。

Plan hash value: 3428151562
-----------------------------------------------------------------------------
| Id  | Operation         | Name    | Rows  | Bytes | Cost (%CPU)| Time     |
-----------------------------------------------------------------------------
|   0 | SELECT STATEMENT  |         |     6 |   678 |     9   (0)| 00:00:01 |
|*  1 |  TABLE ACCESS FULL| STAGING |     6 |   678 |     9   (0)| 00:00:01 |
-----------------------------------------------------------------------------
Predicate Information (identified by operation id):
---------------------------------------------------
   1 - filter(UPPER("S"."EXTRACTED")='N')
4

1 回答 1

0

好的，所以我更改了您的代码，尽量避免这些事情select count(1) into rec_num from staging，因为 oracle 在这里所做的它将 pl/sql 转换为 sql 并将 sql 转换为 pl/sql 所以更好地使用游标，这是更改后的代码。并尝试添加索引create index extracted_idx on staging(extracted asc);，所以这里是修改后的代码：我不知道它是否可以编译，因为我无法检查它，但它应该可以工作。

CREATE OR REPLACE PROCEDURE PKG_EXTRACT(l_brand IN VARCHAR2) AS

  l_file      UTL_FILE.FILE_TYPE;
  l_buffer    RAW(32767);
  l_amount    BINARY_INTEGER := 32767;
  l_pos       INTEGER;
  l_blob      BLOB;
  l_blob_len  INTEGER;
  x           NUMBER;
  l_file_name VARCHAR2(200);
  l_count     INTEGER := 1;
  v_code      NUMBER;
  v_errm      VARCHAR2(64);
  log_file    UTL_FILE.FILE_TYPE;
  rec_num     number;

  CURSOR extract_cur IS
    SELECT DATA, 
               BIN_NAME 
            FROM STAGING 
         WHERE EXTRACTED = 'N';

  cursor c_count is
    select count(1) cnt 
          from staging;

  cursor c_rec_num is
    select count(1)  cnt
         from staging 
        where EXTRACTED = 'N';

BEGIN
  log_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR',
                             'pkg_extract.log',
                             'a',
                             32767);

  UTL_FILE.put_line(log_file,
                    'Logging is being done in 24 hours format - V1.5 ',
                    TRUE);
  UTL_FILE.put_line(log_file,
                    'Extract procedure started on Date-Time = ' ||
                    TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),
                    TRUE);

  open c_cnt;
  fetch c_cnt into rec_num;
  close c_cnt;

  open c_rec_num;
  fetch c_rec_num into rec_num;
  close c_rec_num;

  UTL_FILE.put_line(log_file,
                    'Total Number of records found = ' || rec_num,
                    TRUE);
  --dont know why you doing this again
  open c_rec_num;
  fetch c_rec_num into rec_num;
  close c_rec_num;

  UTL_FILE.put_line(log_file,
                    'Total Number of records matching criteria = ' ||
                    rec_num,
                    TRUE);

  FOR extract_rec IN extract_cur LOOP

    l_pos      := 1;
    l_blob     := extract_rec.DATA;
    l_blob_len := DBMS_LOB.getlength(l_blob);

    -- Save blob length.
    x := l_blob_len;

    l_file_name := extract_rec.BIN_NAME;

    -- Open the destination file.

    l_file := UTL_FILE.fopen('DATA_DOWNLOAD_DIR', l_file_name, 'wb', 32767);

    -- Read chunks of the BLOB and write them to the file until complete.

    WHILE l_pos <= l_blob_len LOOP

      DBMS_LOB.read(l_blob, l_amount, l_pos, l_buffer);

      UTL_FILE.put_raw(l_file, l_buffer, TRUE);

      -- Set the start position for the next cut.
      l_pos := l_pos + l_amount;

    --updating the extract field

    END LOOP;

    l_count := l_count + 1;
    -- Close the file.
    UTL_FILE.fclose(l_file);

    update staging
       set extracted = 'Y', 
                 extract_timestamp = sysdate
     where bin_name = extract_rec.BIN_NAME;
    commit;

  END LOOP;

  UTL_FILE.put_line(log_file,
                    'Extract procedure Completed on Date-Time = ' ||
                    TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'),
                    TRUE);

  IF UTL_FILE.is_open(log_file) THEN
    UTL_FILE.fclose(log_file);
  end if;
END;

EXCEPTION
WHEN OTHERS THEN 
        v_code := SQLCODE; v_errm := SUBSTR(SQLERRM, 1, 64); 
        UTL_FILE.put_line(log_file, '--------------------------------------', TRUE); 
        UTL_FILE.put_line(log_file, 'Error Occurred while executing ' || 'Error code ' || v_code || ': ' || v_errm, TRUE);
        UTL_FILE.put_line(log_file, 'Extract procedure Completed with errors - ' || TO_TIMESTAMP(LOCALTIMESTAMP, 'DD-MON-RR,HH24.MI.SSXFF'), TRUE); 
        UTL_FILE.put_line(log_file, '--------------------------------------', TRUE);

  -- Close the file if something goes wrong.
    IF UTL_FILE.is_open(l_file) THEN

    UTL_FILE.fclose(l_file);

        IF UTL_FILE.is_open(log_file) THEN 
            UTL_FILE.fclose(log_file);
        end if;
    END IF; 
END;
/
于 2016-12-13T07:56:03.823 回答