【问题标题】:T SQL multi thread for launching xp_cmdshell jobs用于启动 xp_cmdshell 作业的 T SQL 多线程
【发布时间】:2013-04-09 06:19:51
【问题描述】:

我的工作是获取数据库备份文件并使用七个 zips 命令行实用程序将它们压缩为 .7z 文件,目前它需要大约 8 个小时来运行所有 .bak 文件,因为它一次只做一个.这是在大约 16 个内核上运行的,而 7z 进程似乎只使用 1 个内核,所以我希望能够运行 xp_cmdshell 命令的多个实例来一次压缩多个文件。有没有办法在 MSSQL Server 2005 上执行 T SQL 中的命令列表?

我在下面发布了我的脚本。

这是我用来压缩文件的程序的链接。 [http://downloads.sourceforge.net/sevenzip/7za920.zip][1]

-- YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
-- II       zip all files in a folder                       II
-- VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV

-- Zip and then Delete all files from the backup folder
-- List all files in a directory - T-SQL parse string for date and filename 
DECLARE  @PathName       VARCHAR(256) , 
         @CMD            VARCHAR(512) 

CREATE TABLE #CommandShell ( Line VARCHAR(512)) 

    -- To use the xp_cmdshell option it has to be enabled. you can use the script bellow to enable it.
    --
    -- -- run exec sp_configure to see if the option exists in this list and you can check if it is enabled.
    -- EXEC sp_configure
    -- go
    -- -- if you don't see xp_cmdshell in the list then you will have to enable advanced options first 
    -- -- before you can enable the xp_cmdshell option.
    -- EXEC sp_configure 'show advanced options', 1;
    -- go
    -- reconfigure
    -- go
    -- -- if xp_cmdshell is in the list then you should just need to run this script.
    -- exec sp_configure 'xp_cmdshell', 1
    -- go
    -- reconfigure

SET @PathName = 'D:\FILES\Backups\' 

SET @CMD = 'DIR ' + @PathName + ' /TC' 

INSERT INTO #CommandShell 
EXEC MASTER..xp_cmdshell   @CMD 

-- Delete lines not containing filename
DELETE 
FROM   #CommandShell 
WHERE  Line NOT LIKE '[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] %' 
OR Line LIKE '%<DIR>%'
OR Line is null

-- SQL reverse string function - charindex string function 
SELECT ROW_NUMBER() OVER (ORDER BY REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) )) AS ROW_NUM,
  FileName = REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) ),
  CreateDate = LEFT(Line,10)
INTO #tempFileList 
FROM #CommandShell
WHERE REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) ) LIKE '%.bak'
ORDER BY FileName

DROP TABLE #CommandShell

DECLARE @FileMaxRownum INT
SET @FileMaxRownum = (SELECT MAX(ROW_NUM) FROM #tempFileList)

DECLARE @FileIter INT
SET @FileIter = (SELECT MIN(ROW_NUM) FROM #tempFileList)

WHILE @FileIter <= @FileMaxRownum
BEGIN
    DECLARE @DelFile varchar(200)
    --@@
    DECLARE @cmd2 VARCHAR(1000)
    SET @cmd2 = null
    DECLARE @db_bkp_files_dir varchar(100)
    SET @db_bkp_files_dir = null
    DECLARE @archive_destination_dir varchar(100)
    SET @archive_destination_dir = null
    DECLARE @archive_name varchar(100)
    SET @archive_name = null
    DECLARE @7z_path varchar(100)
    SET @7z_path = null
    set @archive_destination_dir = @PathName --destination dir
    set @7z_path = 'D:\FILES'
    set @db_bkp_files_dir = right(@PathName,1) --db backup files origin 
    SELECT TOP(1) @archive_name = FileName FROM #tempFileList WHERE ROW_NUM = @FileIter
    SET @cmd2 = @7z_path + '\7za a -t7z -mx5 -ms=off ' + @archive_destination_dir + @archive_name + '.7z ' +  @archive_destination_dir + @archive_name
    print @cmd2
    EXEC xp_cmdshell @cmd2
    --@@
    SELECT TOP(1) @DelFile = 'del ' + @PathName + FileName FROM #tempFileList WHERE ROW_NUM = @FileIter
    EXEC xp_cmdshell @DelFile
    SET @FileIter = @FileIter + 1
END
DROP TABLE #tempFileList

【问题讨论】:

    标签: sql multithreading xp-cmdshell


    【解决方案1】:

    我认为问题不在于 SQL Server。如果您想同时运行多个执行线程,您可以使用 SQL Server 代理中的计划作业来执行此操作...但是,这只是您正在使用的 7-Zip 中的压缩算法的一种解决方法。

    The default compression won't use more than two cores, but an alternate compression algorithm, such as b-zip, will make better use of your server's available cores.

    不要使用-t7z 开关,而是尝试使用-tbzip2,这是上面博客文章中详细介绍的算法。

    SET @cmd2 = @7z_path + '\7za a -tbzip2 -mx5 -ms=off ' 
      + @archive_destination_dir 
      + @archive_name + '.7z ' 
      + @archive_destination_dir 
      + @archive_name
    

    【讨论】:

    • 我测试了 -tbzip 选项(必须删除“-ms=off”选项才能使其工作),它的压缩速度更快,但最后文件更大,所以我最终使用了 -t7z 选项,并与您的其他建议一起为每个文件创建服务器代理作业,因为我有大约 30 个不同的数据库,所以这个选项仍然给了我相当好的速度并使用了服务器上的所有核心。跨度>
    【解决方案2】:

    这是我最终用来测试加速进程的脚本。它完成后将作业留在 msdb 数据库中,因此我必须返回并删除所有在它们完成运行后添加的作业,所以它有点混乱,如果我还没有为服务器代理设置权限服务和我正在访问的文件夹我认为它会给我带来错误。 我可能会重写它并为每个数据库创建单独的备份和压缩作业,所以我不会一直删除和创建服务器代理作业,但这适用于测试。

    -- YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
    -- II       zip all files in a folder                       II
    -- VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV
    
    -- Zip and then Delete all files from the backup folder
    -- List all files in a directory - T-SQL parse string for date and filename 
    DECLARE  @PathName       VARCHAR(256) , 
             @CMD            VARCHAR(512) 
    
    CREATE TABLE #CommandShell ( Line VARCHAR(512)) 
    
    SET @PathName = 'D:\FILES\Backups\' 
    
    SET @CMD = 'DIR ' + @PathName + ' /TC' 
    
    INSERT INTO #CommandShell 
    EXEC MASTER..xp_cmdshell   @CMD 
    
    -- Delete lines not containing filename
    DELETE 
    FROM   #CommandShell 
    WHERE  Line NOT LIKE '[0-9][0-9]/[0-9][0-9]/[0-9][0-9][0-9][0-9] %' 
    OR Line LIKE '%<DIR>%'
    OR Line is null
    
    -- SQL reverse string function - charindex string function 
    SELECT ROW_NUMBER() OVER (ORDER BY REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) )) AS ROW_NUM,
      FileName = REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) ),
      CreateDate = LEFT(Line,10)
    INTO #tempFileList 
    FROM #CommandShell
    WHERE REVERSE( LEFT(REVERSE(Line),CHARINDEX(' ',REVERSE(line))-1 ) ) LIKE '%.bak'
    ORDER BY FileName
    
    DROP TABLE #CommandShell
    
    DECLARE @FileMaxRownum INT
    SET @FileMaxRownum = (SELECT MAX(ROW_NUM) FROM #tempFileList)
    
    DECLARE @FileIter INT
    SET @FileIter = (SELECT MIN(ROW_NUM) FROM #tempFileList)
    
    WHILE @FileIter <= @FileMaxRownum
    BEGIN
        DECLARE @DelFile varchar(200)
        --@@
        DECLARE @cmd2 VARCHAR(1000)
        SET @cmd2 = null
        DECLARE @db_bkp_files_dir varchar(100)
        SET @db_bkp_files_dir = null
        DECLARE @archive_destination_dir varchar(100)
        SET @archive_destination_dir = null
        DECLARE @archive_name varchar(100)
        SET @archive_name = null
        DECLARE @7z_path varchar(100)
        SET @7z_path = null
        set @archive_destination_dir = @PathName --destination dir
        set @7z_path = 'D:\FILES'
        set @db_bkp_files_dir = right(@PathName,1) --db backup files origin 
        SELECT TOP(1) @archive_name = FileName FROM #tempFileList WHERE ROW_NUM = @FileIter
        --   7za switches: 
        --   -mx0 (Don't compress)
        --   -mx1 (Low)
        --   -mx3 (Fast)
        --   -mx5 (Normal)
        --   -mx7 (Maximum)
        --   -mx9 (Ultra)
        --
        --   -mmt (enable multithreading)
        --   
        --   -t7z
        --   -tbzip2
        SET @cmd2 = @7z_path + '\7za a -t7z -mx5 -ms=off ' + @archive_destination_dir + @archive_name + '.7z ' +  @archive_destination_dir + @archive_name
        --SET @cmd2 = @7z_path + '\7za a -tbzip2 -mx5 ' + @archive_destination_dir + @archive_name + '.7z ' +  @archive_destination_dir + @archive_name
    
        DECLARE @cmd3 nvarchar(max)
        SET @cmd3 = 'USE msdb; EXEC sp_add_job @job_name = N''' + @archive_name + ' Zip job' + ''''
    
        DECLARE @cmd4 nvarchar(max)
        SET @cmd4 = 'USE msdb; EXEC sp_add_jobstep @job_name = N''' + @archive_name + ' Zip job' +''', @step_name = N''' + @archive_name + ' Zip job Step 1'  + ''', ' +
                    '@subsystem = N''TSQL'', @command = N''' + 'DECLARE @cmd VARCHAR(1000) SET @cmd = ''''' + @7z_path + '\7za a -t7z -mx5 -ms=off ' + @archive_destination_dir + @archive_name + '.7z ' +  @archive_destination_dir + @archive_name + ''''' EXEC xp_cmdshell @cmd '  + ''', @retry_attempts = 5, @retry_interval = 1'
    
        DECLARE @cmd5 nvarchar(max)
        SET @cmd5 = 'USE msdb; EXEC dbo.sp_add_jobserver @job_name = N''' + @archive_name + ' Zip job' + ''', @server_name = N''ServerName'';'
    
        DECLARE @cmd6 nvarchar(max)
        SET @cmd6 = 'USE msdb; EXEC dbo.sp_start_job N''' + @archive_name + ' Zip job' + ''';'
    
        DECLARE @cmd7 nvarchar(max)
        SET @cmd7 = 'EXEC sp_delete_job @job_name = N''' + @archive_name + ' Zip job' + ''';'
    
        PRINT @cmd3
        EXEC sp_executesql @statement = @cmd3
        PRINT @cmd4
        EXEC sp_executesql @statement = @cmd4
        PRINT @cmd5
        EXEC sp_executesql @statement = @cmd5
        PRINT @cmd6
        EXEC sp_executesql @statement = @cmd6
    
        PRINT @cmd7
    
        --print @cmd2
        --EXEC xp_cmdshell @cmd2
        --@@
        SELECT TOP(1) @DelFile = 'del ' + @PathName + FileName FROM #tempFileList WHERE ROW_NUM = @FileIter
        --EXEC xp_cmdshell @DelFile
        SET @FileIter = @FileIter + 1
    END
    DROP TABLE #tempFileList
    

    【讨论】:

      【解决方案3】:

      sp_add_job 有一个@delete_level 参数,它允许在成功完成后自动删除作业。

      http://technet.microsoft.com/en-us/library/aa259577(v=sql.80).aspx

      【讨论】:

      • 请避免发布仅链接的答案
      猜你喜欢
      • 2012-06-17
      • 2021-05-08
      • 1970-01-01
      • 1970-01-01
      • 1970-01-01
      • 2013-03-15
      • 1970-01-01
      • 1970-01-01
      • 1970-01-01
      相关资源
      最近更新 更多