如何在sql中批量选择大表中的记录

wtlkbnrh  于 2021-07-29  发布在  Java
关注(0)|答案(1)|浏览(343)

我有一个包含数十亿条记录的表。我想用c# while 循环。我曾经 Offset 以及 Fetch Next ,但要得到结果需要太长时间。如果表中包含的记录数低于10万条,则可以正常工作。批量选择记录的最佳方法是什么?
样本代码

int Count  = 500000 // Select Count(id) from table
int OffSet = 1; //Next Time 100000
int FetchNext = 100000; //Next Time 200000

 while (OffSet < Count)
{
   _strsqlcommand = "Select id from table ORDER BY id  OFFSET " + OffSet + " ROWS  FETCH NEXT " + FetchNext + " ROWS ONLY"

_sqlCommand = new SqlCommand(_strsqlcommand, _sqlConnection);
_sqlDataReader = _sqlCommand.ExecuteReader();

 OffSet += FetchNext;
}
xqkwcwgp

xqkwcwgp1#

如果您的id是一个数字,并且您的网络带宽足够好:
首先,找出最大值和最小值 Id .
其次,在where子句中创建一系列数字(对许多记录进行排序是一项高成本的操作)。使用 Where Id > @A and Id<@B 比…快得多 fetch next ).
使用tpl从数据库中提取数据。

namespace BatchLoad
{
    class Program
    {
        static void Main(string[] args)
        {

            var start = DateTime.Now;

            int MinId = 169328112;
            int MaxId = 505690099;

            int BatchCount = 1000000;
            List<QueryExe> Alllist=new List<QueryExe>(BatchCount);
            var stack =new  ConcurrentStack<int>();

            int i = MinId;
            int index = 0;
            while(i<MaxId+1)
            {
                int minid = i;
                int maxid = i + BatchCount;
                string q = $"SELECT  [Id]   FROM YourTable with(nolock)  WHERE Id>={minid} and Id<{maxid} ";
                string c =    "Data Source=.;Initial Catalog=YourDatabase;Persist Security Info=True;User ID=??;Password=!!!;MultipleActiveResultSets=True";
                i = maxid;
                Alllist.Add(new QueryExe(q,c, index));
                index++;
            }

            long ProccessCount = 0;
            Parallel.ForEach(Alllist, new ParallelOptions {MaxDegreeOfParallelism = 100}, command =>
            {

                Task.Yield();
                var temp = command.GetId();
                if(temp?.Count>0)stack.PushRange(temp.ToArray());
                Interlocked.Add(ref ProccessCount,1);
                var donitems = Interlocked.Read(ref ProccessCount);
                if (donitems %10 == 0)
                {
                    Console.WriteLine($"{donitems} / {Alllist.Count}  TotalM={stack.Count/1000000} Total={stack.Count}");

                }

            });
            GC.Collect();
            Console.WriteLine($"Done>{DateTime.Now.Subtract(start).TotalMilliseconds} count ={stack.Count/1000000}");
            Console.ReadLine();

        }

    }

    public class QueryExe
    {
        private string Q = "";
        private string C = "";
        private int i = 0;
        public QueryExecutor(string Q, string C, int i)
        {
            this.Q = Q;
            this.C = C;
            this.i = i;
        }

        public List<int> GetId()
        {
            var result = new List<int>();

            try
            {
                SqlConnection conn = new SqlConnection(C);
                SqlCommand command = new SqlCommand(this.Q, conn);
                command.CommandTimeout = 180;
                using (conn)
                {
                    conn.Open();
                    using (SqlDataReader reader = command.ExecuteReader())
                    {

                        while (reader.Read())
                        {
                            result.Add(reader.GetInt32(0));
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Exception>{i}");
            }

            return result;
        }

    }
}

相关问题