[{"data":1,"prerenderedAt":1871},["ShallowReactive",2],{"doc-\u002Fexamples\u002Fpython-remove-duplicates-from-data-example":3},{"id":4,"title":5,"body":6,"description":1864,"extension":1865,"meta":1866,"navigation":266,"path":1867,"seo":1868,"stem":1869,"__hash__":1870},"content\u002Fexamples\u002Fpython-remove-duplicates-from-data-example.md","Python Remove Duplicates from Data Example",{"type":7,"value":8,"toc":1828},"minimark",[9,13,17,20,33,38,169,176,180,183,194,202,206,209,298,301,338,343,361,365,373,385,388,477,481,486,579,582,616,620,626,633,637,648,655,659,662,797,799,831,835,855,858,861,1034,1036,1069,1073,1079,1090,1094,1105,1108,1119,1123,1130,1141,1144,1164,1167,1270,1272,1297,1304,1308,1311,1318,1321,1386,1389,1416,1422,1425,1484,1487,1495,1498,1549,1551,1576,1579,1644,1650,1653,1714,1717,1720,1743,1747,1751,1761,1768,1771,1775,1781,1788,1791,1795,1824],[10,11,5],"h1",{"id":12},"python-remove-duplicates-from-data-example",[14,15,16],"p",{},"This beginner-friendly example shows simple ways to remove duplicate values from Python data.",[14,18,19],{},"The focus is on lists, because that is where this problem appears most often. You will see:",[21,22,23,27,30],"ul",{},[24,25,26],"li",{},"how to remove repeated items quickly",[24,28,29],{},"how to keep the original order when needed",[24,31,32],{},"how to do the same job step by step with a loop",[34,35,37],"h2",{"id":36},"quick-answer","Quick answer",[39,40,45],"pre",{"className":41,"code":42,"language":43,"meta":44,"style":44},"language-python shiki shiki-themes material-theme-lighter github-light github-dark","data = [\"apple\", \"banana\", \"apple\", \"orange\", \"banana\"]\nunique_data = list(dict.fromkeys(data))\nprint(unique_data)\n# ['apple', 'banana', 'orange']\n","python","",[46,47,48,114,147,162],"code",{"__ignoreMap":44},[49,50,53,57,61,65,69,73,75,78,81,84,86,88,90,92,94,96,98,101,103,105,107,109,111],"span",{"class":51,"line":52},"line",1,[49,54,56],{"class":55},"su5hD","data ",[49,58,60],{"class":59},"smGrS","=",[49,62,64],{"class":63},"sP7_E"," [",[49,66,68],{"class":67},"sjJ54","\"",[49,70,72],{"class":71},"s_sjI","apple",[49,74,68],{"class":67},[49,76,77],{"class":63},",",[49,79,80],{"class":67}," \"",[49,82,83],{"class":71},"banana",[49,85,68],{"class":67},[49,87,77],{"class":63},[49,89,80],{"class":67},[49,91,72],{"class":71},[49,93,68],{"class":67},[49,95,77],{"class":63},[49,97,80],{"class":67},[49,99,100],{"class":71},"orange",[49,102,68],{"class":67},[49,104,77],{"class":63},[49,106,80],{"class":67},[49,108,83],{"class":71},[49,110,68],{"class":67},[49,112,113],{"class":63},"]\n",[49,115,117,120,122,126,129,132,135,139,141,144],{"class":51,"line":116},2,[49,118,119],{"class":55},"unique_data ",[49,121,60],{"class":59},[49,123,125],{"class":124},"sZMiF"," list",[49,127,128],{"class":63},"(",[49,130,131],{"class":124},"dict",[49,133,134],{"class":63},".",[49,136,138],{"class":137},"slqww","fromkeys",[49,140,128],{"class":63},[49,142,143],{"class":137},"data",[49,145,146],{"class":63},"))\n",[49,148,150,154,156,159],{"class":51,"line":149},3,[49,151,153],{"class":152},"sptTA","print",[49,155,128],{"class":63},[49,157,158],{"class":137},"unique_data",[49,160,161],{"class":63},")\n",[49,163,165],{"class":51,"line":164},4,[49,166,168],{"class":167},"sutJx","# ['apple', 'banana', 'orange']\n",[14,170,171,172,175],{},"Use ",[46,173,174],{},"dict.fromkeys()"," when you want to remove duplicates and keep the original order.",[34,177,179],{"id":178},"what-this-example-solves","What this example solves",[14,181,182],{},"This example is useful when you need to:",[21,184,185,188,191],{},[24,186,187],{},"remove repeated items from a list",[24,189,190],{},"understand the difference between removing duplicates with and without keeping order",[24,192,193],{},"learn a practical pattern without reading a full theory lesson",[14,195,196,197,134],{},"If you want a task-focused guide, see ",[198,199,201],"a",{"href":200},"\u002Fhow-to\u002Fhow-to-remove-duplicates-from-a-list-in-python\u002F","how to remove duplicates from a list in Python",[34,203,205],{"id":204},"example-1-remove-duplicates-with-a-set","Example 1: Remove duplicates with a set",[14,207,208],{},"The shortest way to remove duplicates is to turn the list into a set.",[39,210,212],{"className":41,"code":211,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\", \"orange\", \"banana\"]\n\nunique_data = set(data)\n\nprint(unique_data)\n",[46,213,214,262,268,283,287],{"__ignoreMap":44},[49,215,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,256,258,260],{"class":51,"line":52},[49,217,56],{"class":55},[49,219,60],{"class":59},[49,221,64],{"class":63},[49,223,68],{"class":67},[49,225,72],{"class":71},[49,227,68],{"class":67},[49,229,77],{"class":63},[49,231,80],{"class":67},[49,233,83],{"class":71},[49,235,68],{"class":67},[49,237,77],{"class":63},[49,239,80],{"class":67},[49,241,72],{"class":71},[49,243,68],{"class":67},[49,245,77],{"class":63},[49,247,80],{"class":67},[49,249,100],{"class":71},[49,251,68],{"class":67},[49,253,77],{"class":63},[49,255,80],{"class":67},[49,257,83],{"class":71},[49,259,68],{"class":67},[49,261,113],{"class":63},[49,263,264],{"class":51,"line":116},[49,265,267],{"emptyLinePlaceholder":266},true,"\n",[49,269,270,272,274,277,279,281],{"class":51,"line":149},[49,271,119],{"class":55},[49,273,60],{"class":59},[49,275,276],{"class":124}," set",[49,278,128],{"class":63},[49,280,143],{"class":137},[49,282,161],{"class":63},[49,284,285],{"class":51,"line":164},[49,286,267],{"emptyLinePlaceholder":266},[49,288,290,292,294,296],{"class":51,"line":289},5,[49,291,153],{"class":152},[49,293,128],{"class":63},[49,295,158],{"class":137},[49,297,161],{"class":63},[14,299,300],{},"Possible output:",[39,302,304],{"className":41,"code":303,"language":43,"meta":44,"style":44},"{'banana', 'orange', 'apple'}\n",[46,305,306],{"__ignoreMap":44},[49,307,308,311,314,316,318,320,323,325,327,329,331,333,335],{"class":51,"line":52},[49,309,310],{"class":63},"{",[49,312,313],{"class":67},"'",[49,315,83],{"class":71},[49,317,313],{"class":67},[49,319,77],{"class":63},[49,321,322],{"class":67}," '",[49,324,100],{"class":71},[49,326,313],{"class":67},[49,328,77],{"class":63},[49,330,322],{"class":67},[49,332,72],{"class":71},[49,334,313],{"class":67},[49,336,337],{"class":63},"}\n",[339,340,342],"h3",{"id":341},"what-this-does","What this does",[21,344,345,351,354],{},[24,346,347,350],{},[46,348,349],{},"set(data)"," removes duplicate values",[24,352,353],{},"each value appears only once in the result",[24,355,356,357,360],{},"the result is a ",[46,358,359],{},"set",", not a list",[339,362,364],{"id":363},"important-note","Important note",[14,366,367,368,372],{},"A set does ",[369,370,371],"strong",{},"not"," keep the original order of the list. That means this method is best when order does not matter.",[14,374,375,376,380,381,134],{},"If you want to learn more about sets, read ",[198,377,379],{"href":378},"\u002Flearn\u002Fpython-sets-explained\u002F","Python sets explained"," or ",[198,382,384],{"href":383},"\u002Freference\u002Fpython-set-creating-a-set\u002F","Python set: creating a set",[14,386,387],{},"If you need the result as a list, convert it back:",[39,389,391],{"className":41,"code":390,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\", \"orange\", \"banana\"]\n\nunique_data = list(set(data))\n\nprint(unique_data)\n",[46,392,393,441,445,463,467],{"__ignoreMap":44},[49,394,395,397,399,401,403,405,407,409,411,413,415,417,419,421,423,425,427,429,431,433,435,437,439],{"class":51,"line":52},[49,396,56],{"class":55},[49,398,60],{"class":59},[49,400,64],{"class":63},[49,402,68],{"class":67},[49,404,72],{"class":71},[49,406,68],{"class":67},[49,408,77],{"class":63},[49,410,80],{"class":67},[49,412,83],{"class":71},[49,414,68],{"class":67},[49,416,77],{"class":63},[49,418,80],{"class":67},[49,420,72],{"class":71},[49,422,68],{"class":67},[49,424,77],{"class":63},[49,426,80],{"class":67},[49,428,100],{"class":71},[49,430,68],{"class":67},[49,432,77],{"class":63},[49,434,80],{"class":67},[49,436,83],{"class":71},[49,438,68],{"class":67},[49,440,113],{"class":63},[49,442,443],{"class":51,"line":116},[49,444,267],{"emptyLinePlaceholder":266},[49,446,447,449,451,453,455,457,459,461],{"class":51,"line":149},[49,448,119],{"class":55},[49,450,60],{"class":59},[49,452,125],{"class":124},[49,454,128],{"class":63},[49,456,359],{"class":124},[49,458,128],{"class":63},[49,460,143],{"class":137},[49,462,146],{"class":63},[49,464,465],{"class":51,"line":164},[49,466,267],{"emptyLinePlaceholder":266},[49,468,469,471,473,475],{"class":51,"line":289},[49,470,153],{"class":152},[49,472,128],{"class":63},[49,474,158],{"class":137},[49,476,161],{"class":63},[34,478,480],{"id":479},"example-2-remove-duplicates-and-keep-order","Example 2: Remove duplicates and keep order",[14,482,483,484,134],{},"If you want to keep the first appearance of each item, use ",[46,485,174],{},[39,487,489],{"className":41,"code":488,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\", \"orange\", \"banana\"]\n\nunique_data = list(dict.fromkeys(data))\n\nprint(unique_data)\n",[46,490,491,539,543,565,569],{"__ignoreMap":44},[49,492,493,495,497,499,501,503,505,507,509,511,513,515,517,519,521,523,525,527,529,531,533,535,537],{"class":51,"line":52},[49,494,56],{"class":55},[49,496,60],{"class":59},[49,498,64],{"class":63},[49,500,68],{"class":67},[49,502,72],{"class":71},[49,504,68],{"class":67},[49,506,77],{"class":63},[49,508,80],{"class":67},[49,510,83],{"class":71},[49,512,68],{"class":67},[49,514,77],{"class":63},[49,516,80],{"class":67},[49,518,72],{"class":71},[49,520,68],{"class":67},[49,522,77],{"class":63},[49,524,80],{"class":67},[49,526,100],{"class":71},[49,528,68],{"class":67},[49,530,77],{"class":63},[49,532,80],{"class":67},[49,534,83],{"class":71},[49,536,68],{"class":67},[49,538,113],{"class":63},[49,540,541],{"class":51,"line":116},[49,542,267],{"emptyLinePlaceholder":266},[49,544,545,547,549,551,553,555,557,559,561,563],{"class":51,"line":149},[49,546,119],{"class":55},[49,548,60],{"class":59},[49,550,125],{"class":124},[49,552,128],{"class":63},[49,554,131],{"class":124},[49,556,134],{"class":63},[49,558,138],{"class":137},[49,560,128],{"class":63},[49,562,143],{"class":137},[49,564,146],{"class":63},[49,566,567],{"class":51,"line":164},[49,568,267],{"emptyLinePlaceholder":266},[49,570,571,573,575,577],{"class":51,"line":289},[49,572,153],{"class":152},[49,574,128],{"class":63},[49,576,158],{"class":137},[49,578,161],{"class":63},[14,580,581],{},"Output:",[39,583,585],{"className":41,"code":584,"language":43,"meta":44,"style":44},"['apple', 'banana', 'orange']\n",[46,586,587],{"__ignoreMap":44},[49,588,589,592,594,596,598,600,602,604,606,608,610,612,614],{"class":51,"line":52},[49,590,591],{"class":63},"[",[49,593,313],{"class":67},[49,595,72],{"class":71},[49,597,313],{"class":67},[49,599,77],{"class":63},[49,601,322],{"class":67},[49,603,83],{"class":71},[49,605,313],{"class":67},[49,607,77],{"class":63},[49,609,322],{"class":67},[49,611,100],{"class":71},[49,613,313],{"class":67},[49,615,113],{"class":63},[339,617,619],{"id":618},"why-this-works","Why this works",[14,621,622,625],{},[46,623,624],{},"dict.fromkeys(data)"," creates a dictionary using each list item as a key.",[14,627,628,629,632],{},"Dictionary keys must be unique, so duplicates are removed automatically. Then ",[46,630,631],{},"list(...)"," turns those keys back into a list.",[339,634,636],{"id":635},"why-beginners-like-this-method","Why beginners like this method",[21,638,639,642,645],{},[24,640,641],{},"it is short",[24,643,644],{},"it keeps order",[24,646,647],{},"it works well for common cleanup tasks",[14,649,650,651,134],{},"If dictionaries are new to you, see ",[198,652,654],{"href":653},"\u002Flearn\u002Fpython-dictionaries-explained\u002F","Python dictionaries explained",[34,656,658],{"id":657},"example-3-build-a-unique-list-with-a-loop","Example 3: Build a unique list with a loop",[14,660,661],{},"A loop is longer, but it helps you understand the logic step by step.",[39,663,665],{"className":41,"code":664,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\", \"orange\", \"banana\"]\nunique_data = []\n\nfor item in data:\n    if item not in unique_data:\n        unique_data.append(item)\n\nprint(unique_data)\n",[46,666,667,715,724,728,746,763,781,786],{"__ignoreMap":44},[49,668,669,671,673,675,677,679,681,683,685,687,689,691,693,695,697,699,701,703,705,707,709,711,713],{"class":51,"line":52},[49,670,56],{"class":55},[49,672,60],{"class":59},[49,674,64],{"class":63},[49,676,68],{"class":67},[49,678,72],{"class":71},[49,680,68],{"class":67},[49,682,77],{"class":63},[49,684,80],{"class":67},[49,686,83],{"class":71},[49,688,68],{"class":67},[49,690,77],{"class":63},[49,692,80],{"class":67},[49,694,72],{"class":71},[49,696,68],{"class":67},[49,698,77],{"class":63},[49,700,80],{"class":67},[49,702,100],{"class":71},[49,704,68],{"class":67},[49,706,77],{"class":63},[49,708,80],{"class":67},[49,710,83],{"class":71},[49,712,68],{"class":67},[49,714,113],{"class":63},[49,716,717,719,721],{"class":51,"line":116},[49,718,119],{"class":55},[49,720,60],{"class":59},[49,722,723],{"class":63}," []\n",[49,725,726],{"class":51,"line":149},[49,727,267],{"emptyLinePlaceholder":266},[49,729,730,734,737,740,743],{"class":51,"line":164},[49,731,733],{"class":732},"sVHd0","for",[49,735,736],{"class":55}," item ",[49,738,739],{"class":732},"in",[49,741,742],{"class":55}," data",[49,744,745],{"class":63},":\n",[49,747,748,751,753,755,758,761],{"class":51,"line":289},[49,749,750],{"class":732},"    if",[49,752,736],{"class":55},[49,754,371],{"class":59},[49,756,757],{"class":59}," in",[49,759,760],{"class":55}," unique_data",[49,762,745],{"class":63},[49,764,766,769,771,774,776,779],{"class":51,"line":765},6,[49,767,768],{"class":55},"        unique_data",[49,770,134],{"class":63},[49,772,773],{"class":137},"append",[49,775,128],{"class":63},[49,777,778],{"class":137},"item",[49,780,161],{"class":63},[49,782,784],{"class":51,"line":783},7,[49,785,267],{"emptyLinePlaceholder":266},[49,787,789,791,793,795],{"class":51,"line":788},8,[49,790,153],{"class":152},[49,792,128],{"class":63},[49,794,158],{"class":137},[49,796,161],{"class":63},[14,798,581],{},[39,800,801],{"className":41,"code":584,"language":43,"meta":44,"style":44},[46,802,803],{"__ignoreMap":44},[49,804,805,807,809,811,813,815,817,819,821,823,825,827,829],{"class":51,"line":52},[49,806,591],{"class":63},[49,808,313],{"class":67},[49,810,72],{"class":71},[49,812,313],{"class":67},[49,814,77],{"class":63},[49,816,322],{"class":67},[49,818,83],{"class":71},[49,820,313],{"class":67},[49,822,77],{"class":63},[49,824,322],{"class":67},[49,826,100],{"class":71},[49,828,313],{"class":67},[49,830,113],{"class":63},[339,832,834],{"id":833},"how-it-works","How it works",[21,836,837,842,847,852],{},[24,838,839,840],{},"start with an empty list called ",[46,841,158],{},[24,843,844,845],{},"go through each item in ",[46,846,143],{},[24,848,849,850],{},"check whether the item is already in ",[46,851,158],{},[24,853,854],{},"if not, add it",[14,856,857],{},"This approach is useful when you want custom rules. For example, you might want to ignore case or skip empty values.",[14,859,860],{},"Here is a simple case-insensitive version:",[39,862,864],{"className":41,"code":863,"language":43,"meta":44,"style":44},"data = [\"Apple\", \"banana\", \"apple\", \"Orange\", \"BANANA\"]\nunique_data = []\nseen = set()\n\nfor item in data:\n    normalized = item.lower()\n    if normalized not in seen:\n        seen.add(normalized)\n        unique_data.append(item)\n\nprint(unique_data)\n",[46,865,866,917,925,937,941,953,970,986,1003,1018,1023],{"__ignoreMap":44},[49,867,868,870,872,874,876,879,881,883,885,887,889,891,893,895,897,899,901,904,906,908,910,913,915],{"class":51,"line":52},[49,869,56],{"class":55},[49,871,60],{"class":59},[49,873,64],{"class":63},[49,875,68],{"class":67},[49,877,878],{"class":71},"Apple",[49,880,68],{"class":67},[49,882,77],{"class":63},[49,884,80],{"class":67},[49,886,83],{"class":71},[49,888,68],{"class":67},[49,890,77],{"class":63},[49,892,80],{"class":67},[49,894,72],{"class":71},[49,896,68],{"class":67},[49,898,77],{"class":63},[49,900,80],{"class":67},[49,902,903],{"class":71},"Orange",[49,905,68],{"class":67},[49,907,77],{"class":63},[49,909,80],{"class":67},[49,911,912],{"class":71},"BANANA",[49,914,68],{"class":67},[49,916,113],{"class":63},[49,918,919,921,923],{"class":51,"line":116},[49,920,119],{"class":55},[49,922,60],{"class":59},[49,924,723],{"class":63},[49,926,927,930,932,934],{"class":51,"line":149},[49,928,929],{"class":55},"seen ",[49,931,60],{"class":59},[49,933,276],{"class":124},[49,935,936],{"class":63},"()\n",[49,938,939],{"class":51,"line":164},[49,940,267],{"emptyLinePlaceholder":266},[49,942,943,945,947,949,951],{"class":51,"line":289},[49,944,733],{"class":732},[49,946,736],{"class":55},[49,948,739],{"class":732},[49,950,742],{"class":55},[49,952,745],{"class":63},[49,954,955,958,960,963,965,968],{"class":51,"line":765},[49,956,957],{"class":55},"    normalized ",[49,959,60],{"class":59},[49,961,962],{"class":55}," item",[49,964,134],{"class":63},[49,966,967],{"class":137},"lower",[49,969,936],{"class":63},[49,971,972,974,977,979,981,984],{"class":51,"line":783},[49,973,750],{"class":732},[49,975,976],{"class":55}," normalized ",[49,978,371],{"class":59},[49,980,757],{"class":59},[49,982,983],{"class":55}," seen",[49,985,745],{"class":63},[49,987,988,991,993,996,998,1001],{"class":51,"line":788},[49,989,990],{"class":55},"        seen",[49,992,134],{"class":63},[49,994,995],{"class":137},"add",[49,997,128],{"class":63},[49,999,1000],{"class":137},"normalized",[49,1002,161],{"class":63},[49,1004,1006,1008,1010,1012,1014,1016],{"class":51,"line":1005},9,[49,1007,768],{"class":55},[49,1009,134],{"class":63},[49,1011,773],{"class":137},[49,1013,128],{"class":63},[49,1015,778],{"class":137},[49,1017,161],{"class":63},[49,1019,1021],{"class":51,"line":1020},10,[49,1022,267],{"emptyLinePlaceholder":266},[49,1024,1026,1028,1030,1032],{"class":51,"line":1025},11,[49,1027,153],{"class":152},[49,1029,128],{"class":63},[49,1031,158],{"class":137},[49,1033,161],{"class":63},[14,1035,581],{},[39,1037,1039],{"className":41,"code":1038,"language":43,"meta":44,"style":44},"['Apple', 'banana', 'Orange']\n",[46,1040,1041],{"__ignoreMap":44},[49,1042,1043,1045,1047,1049,1051,1053,1055,1057,1059,1061,1063,1065,1067],{"class":51,"line":52},[49,1044,591],{"class":63},[49,1046,313],{"class":67},[49,1048,878],{"class":71},[49,1050,313],{"class":67},[49,1052,77],{"class":63},[49,1054,322],{"class":67},[49,1056,83],{"class":71},[49,1058,313],{"class":67},[49,1060,77],{"class":63},[49,1062,322],{"class":67},[49,1064,903],{"class":71},[49,1066,313],{"class":67},[49,1068,113],{"class":63},[34,1070,1072],{"id":1071},"when-each-method-is-a-good-choice","When each method is a good choice",[14,1074,171,1075,1078],{},[46,1076,1077],{},"set()"," when:",[21,1080,1081,1084,1087],{},[24,1082,1083],{},"you want the shortest solution",[24,1085,1086],{},"order does not matter",[24,1088,1089],{},"you only need unique values quickly",[14,1091,171,1092,1078],{},[46,1093,174],{},[21,1095,1096,1099,1102],{},[24,1097,1098],{},"you want to preserve the original order",[24,1100,1101],{},"you want a short and readable solution",[24,1103,1104],{},"you are cleaning list data in a beginner script",[14,1106,1107],{},"Use a loop when:",[21,1109,1110,1113,1116],{},[24,1111,1112],{},"you want to learn how duplicate removal works",[24,1114,1115],{},"you need custom rules",[24,1117,1118],{},"you want more control over the logic",[34,1120,1122],{"id":1121},"beginner-notes","Beginner notes",[14,1124,1125,1126,1129],{},"These methods work best with lists that contain ",[369,1127,1128],{},"hashable"," values, such as:",[21,1131,1132,1135,1138],{},[24,1133,1134],{},"strings",[24,1136,1137],{},"numbers",[24,1139,1140],{},"tuples",[14,1142,1143],{},"Be careful with these cases:",[21,1145,1146,1149,1154],{},[24,1147,1148],{},"a list of dictionaries cannot be directly turned into a set",[24,1150,1151,1153],{},[46,1152,174],{}," also needs hashable values",[24,1155,1156,1159,1160,1163],{},[46,1157,1158],{},"\"Apple\""," and ",[46,1161,1162],{},"\"apple\""," are different unless you normalize them first",[14,1165,1166],{},"Example of normalization:",[39,1168,1170],{"className":41,"code":1169,"language":43,"meta":44,"style":44},"data = [\"Apple\", \"apple\", \"BANANA\", \"banana\"]\n\nnormalized_unique = list(dict.fromkeys(item.lower() for item in data))\n\nprint(normalized_unique)\n",[46,1171,1172,1212,1216,1255,1259],{"__ignoreMap":44},[49,1173,1174,1176,1178,1180,1182,1184,1186,1188,1190,1192,1194,1196,1198,1200,1202,1204,1206,1208,1210],{"class":51,"line":52},[49,1175,56],{"class":55},[49,1177,60],{"class":59},[49,1179,64],{"class":63},[49,1181,68],{"class":67},[49,1183,878],{"class":71},[49,1185,68],{"class":67},[49,1187,77],{"class":63},[49,1189,80],{"class":67},[49,1191,72],{"class":71},[49,1193,68],{"class":67},[49,1195,77],{"class":63},[49,1197,80],{"class":67},[49,1199,912],{"class":71},[49,1201,68],{"class":67},[49,1203,77],{"class":63},[49,1205,80],{"class":67},[49,1207,83],{"class":71},[49,1209,68],{"class":67},[49,1211,113],{"class":63},[49,1213,1214],{"class":51,"line":116},[49,1215,267],{"emptyLinePlaceholder":266},[49,1217,1218,1221,1223,1225,1227,1229,1231,1233,1235,1237,1239,1241,1244,1247,1249,1251,1253],{"class":51,"line":149},[49,1219,1220],{"class":55},"normalized_unique ",[49,1222,60],{"class":59},[49,1224,125],{"class":124},[49,1226,128],{"class":63},[49,1228,131],{"class":124},[49,1230,134],{"class":63},[49,1232,138],{"class":137},[49,1234,128],{"class":63},[49,1236,778],{"class":137},[49,1238,134],{"class":63},[49,1240,967],{"class":137},[49,1242,1243],{"class":63},"()",[49,1245,1246],{"class":732}," for",[49,1248,736],{"class":137},[49,1250,739],{"class":732},[49,1252,742],{"class":137},[49,1254,146],{"class":63},[49,1256,1257],{"class":51,"line":164},[49,1258,267],{"emptyLinePlaceholder":266},[49,1260,1261,1263,1265,1268],{"class":51,"line":289},[49,1262,153],{"class":152},[49,1264,128],{"class":63},[49,1266,1267],{"class":137},"normalized_unique",[49,1269,161],{"class":63},[14,1271,581],{},[39,1273,1275],{"className":41,"code":1274,"language":43,"meta":44,"style":44},"['apple', 'banana']\n",[46,1276,1277],{"__ignoreMap":44},[49,1278,1279,1281,1283,1285,1287,1289,1291,1293,1295],{"class":51,"line":52},[49,1280,591],{"class":63},[49,1282,313],{"class":67},[49,1284,72],{"class":71},[49,1286,313],{"class":67},[49,1288,77],{"class":63},[49,1290,322],{"class":67},[49,1292,83],{"class":71},[49,1294,313],{"class":67},[49,1296,113],{"class":63},[14,1298,1299,1300,134],{},"If you are doing broader cleanup work, this may also help: ",[198,1301,1303],{"href":1302},"\u002Fexamples\u002Fpython-data-cleaning-script-example\u002F","Python data cleaning script example",[34,1305,1307],{"id":1306},"common-mistakes","Common mistakes",[14,1309,1310],{},"Here are some common problems beginners run into.",[339,1312,1314,1315,1317],{"id":1313},"using-set-and-expecting-the-original-order-to-stay-the-same","Using ",[46,1316,1077],{}," and expecting the original order to stay the same",[14,1319,1320],{},"This will remove duplicates, but the order may change:",[39,1322,1324],{"className":41,"code":1323,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\", \"orange\"]\nprint(list(set(data)))\n",[46,1325,1326,1366],{"__ignoreMap":44},[49,1327,1328,1330,1332,1334,1336,1338,1340,1342,1344,1346,1348,1350,1352,1354,1356,1358,1360,1362,1364],{"class":51,"line":52},[49,1329,56],{"class":55},[49,1331,60],{"class":59},[49,1333,64],{"class":63},[49,1335,68],{"class":67},[49,1337,72],{"class":71},[49,1339,68],{"class":67},[49,1341,77],{"class":63},[49,1343,80],{"class":67},[49,1345,83],{"class":71},[49,1347,68],{"class":67},[49,1349,77],{"class":63},[49,1351,80],{"class":67},[49,1353,72],{"class":71},[49,1355,68],{"class":67},[49,1357,77],{"class":63},[49,1359,80],{"class":67},[49,1361,100],{"class":71},[49,1363,68],{"class":67},[49,1365,113],{"class":63},[49,1367,1368,1370,1372,1375,1377,1379,1381,1383],{"class":51,"line":116},[49,1369,153],{"class":152},[49,1371,128],{"class":63},[49,1373,1374],{"class":124},"list",[49,1376,128],{"class":63},[49,1378,359],{"class":124},[49,1380,128],{"class":63},[49,1382,143],{"class":137},[49,1384,1385],{"class":63},")))\n",[14,1387,1388],{},"If order matters, use:",[39,1390,1392],{"className":41,"code":1391,"language":43,"meta":44,"style":44},"print(list(dict.fromkeys(data)))\n",[46,1393,1394],{"__ignoreMap":44},[49,1395,1396,1398,1400,1402,1404,1406,1408,1410,1412,1414],{"class":51,"line":52},[49,1397,153],{"class":152},[49,1399,128],{"class":63},[49,1401,1374],{"class":124},[49,1403,128],{"class":63},[49,1405,131],{"class":124},[49,1407,134],{"class":63},[49,1409,138],{"class":137},[49,1411,128],{"class":63},[49,1413,143],{"class":137},[49,1415,1385],{"class":63},[339,1417,1419,1420],{"id":1418},"trying-to-remove-duplicates-from-a-list-of-dictionaries-with-set","Trying to remove duplicates from a list of dictionaries with ",[46,1421,1077],{},[14,1423,1424],{},"This causes an error because dictionaries cannot be added to a set.",[39,1426,1428],{"className":41,"code":1427,"language":43,"meta":44,"style":44},"data = [{\"name\": \"Alice\"}, {\"name\": \"Alice\"}]\n# set(data)  # This will fail\n",[46,1429,1430,1479],{"__ignoreMap":44},[49,1431,1432,1434,1436,1439,1441,1444,1446,1449,1451,1454,1456,1459,1462,1464,1466,1468,1470,1472,1474,1476],{"class":51,"line":52},[49,1433,56],{"class":55},[49,1435,60],{"class":59},[49,1437,1438],{"class":63}," [{",[49,1440,68],{"class":67},[49,1442,1443],{"class":71},"name",[49,1445,68],{"class":67},[49,1447,1448],{"class":63},":",[49,1450,80],{"class":67},[49,1452,1453],{"class":71},"Alice",[49,1455,68],{"class":67},[49,1457,1458],{"class":63},"},",[49,1460,1461],{"class":63}," {",[49,1463,68],{"class":67},[49,1465,1443],{"class":71},[49,1467,68],{"class":67},[49,1469,1448],{"class":63},[49,1471,80],{"class":67},[49,1473,1453],{"class":71},[49,1475,68],{"class":67},[49,1477,1478],{"class":63},"}]\n",[49,1480,1481],{"class":51,"line":116},[49,1482,1483],{"class":167},"# set(data)  # This will fail\n",[14,1485,1486],{},"If you need to work with more complex data, you usually have to compare a specific field yourself with a loop.",[339,1488,1490,1491,1159,1493],{"id":1489},"not-normalizing-values-like-apple-and-apple","Not normalizing values like ",[46,1492,1158],{},[46,1494,1162],{},[14,1496,1497],{},"Python treats them as different strings:",[39,1499,1501],{"className":41,"code":1500,"language":43,"meta":44,"style":44},"data = [\"Apple\", \"apple\"]\nprint(list(dict.fromkeys(data)))\n",[46,1502,1503,1527],{"__ignoreMap":44},[49,1504,1505,1507,1509,1511,1513,1515,1517,1519,1521,1523,1525],{"class":51,"line":52},[49,1506,56],{"class":55},[49,1508,60],{"class":59},[49,1510,64],{"class":63},[49,1512,68],{"class":67},[49,1514,878],{"class":71},[49,1516,68],{"class":67},[49,1518,77],{"class":63},[49,1520,80],{"class":67},[49,1522,72],{"class":71},[49,1524,68],{"class":67},[49,1526,113],{"class":63},[49,1528,1529,1531,1533,1535,1537,1539,1541,1543,1545,1547],{"class":51,"line":116},[49,1530,153],{"class":152},[49,1532,128],{"class":63},[49,1534,1374],{"class":124},[49,1536,128],{"class":63},[49,1538,131],{"class":124},[49,1540,134],{"class":63},[49,1542,138],{"class":137},[49,1544,128],{"class":63},[49,1546,143],{"class":137},[49,1548,1385],{"class":63},[14,1550,581],{},[39,1552,1554],{"className":41,"code":1553,"language":43,"meta":44,"style":44},"['Apple', 'apple']\n",[46,1555,1556],{"__ignoreMap":44},[49,1557,1558,1560,1562,1564,1566,1568,1570,1572,1574],{"class":51,"line":52},[49,1559,591],{"class":63},[49,1561,313],{"class":67},[49,1563,878],{"class":71},[49,1565,313],{"class":67},[49,1567,77],{"class":63},[49,1569,322],{"class":67},[49,1571,72],{"class":71},[49,1573,313],{"class":67},[49,1575,113],{"class":63},[14,1577,1578],{},"Normalize first if needed:",[39,1580,1582],{"className":41,"code":1581,"language":43,"meta":44,"style":44},"data = [\"Apple\", \"apple\"]\nprint(list(dict.fromkeys(item.lower() for item in data)))\n",[46,1583,1584,1608],{"__ignoreMap":44},[49,1585,1586,1588,1590,1592,1594,1596,1598,1600,1602,1604,1606],{"class":51,"line":52},[49,1587,56],{"class":55},[49,1589,60],{"class":59},[49,1591,64],{"class":63},[49,1593,68],{"class":67},[49,1595,878],{"class":71},[49,1597,68],{"class":67},[49,1599,77],{"class":63},[49,1601,80],{"class":67},[49,1603,72],{"class":71},[49,1605,68],{"class":67},[49,1607,113],{"class":63},[49,1609,1610,1612,1614,1616,1618,1620,1622,1624,1626,1628,1630,1632,1634,1636,1638,1640,1642],{"class":51,"line":116},[49,1611,153],{"class":152},[49,1613,128],{"class":63},[49,1615,1374],{"class":124},[49,1617,128],{"class":63},[49,1619,131],{"class":124},[49,1621,134],{"class":63},[49,1623,138],{"class":137},[49,1625,128],{"class":63},[49,1627,778],{"class":137},[49,1629,134],{"class":63},[49,1631,967],{"class":137},[49,1633,1243],{"class":63},[49,1635,1246],{"class":732},[49,1637,736],{"class":137},[49,1639,739],{"class":732},[49,1641,742],{"class":137},[49,1643,1385],{"class":63},[339,1645,1647,1648],{"id":1646},"forgetting-to-convert-the-result-back-to-a-list-after-using-set","Forgetting to convert the result back to a list after using ",[46,1649,1077],{},[14,1651,1652],{},"This:",[39,1654,1656],{"className":41,"code":1655,"language":43,"meta":44,"style":44},"data = [\"apple\", \"banana\", \"apple\"]\nunique_data = set(data)\nprint(unique_data)\n",[46,1657,1658,1690,1704],{"__ignoreMap":44},[49,1659,1660,1662,1664,1666,1668,1670,1672,1674,1676,1678,1680,1682,1684,1686,1688],{"class":51,"line":52},[49,1661,56],{"class":55},[49,1663,60],{"class":59},[49,1665,64],{"class":63},[49,1667,68],{"class":67},[49,1669,72],{"class":71},[49,1671,68],{"class":67},[49,1673,77],{"class":63},[49,1675,80],{"class":67},[49,1677,83],{"class":71},[49,1679,68],{"class":67},[49,1681,77],{"class":63},[49,1683,80],{"class":67},[49,1685,72],{"class":71},[49,1687,68],{"class":67},[49,1689,113],{"class":63},[49,1691,1692,1694,1696,1698,1700,1702],{"class":51,"line":116},[49,1693,119],{"class":55},[49,1695,60],{"class":59},[49,1697,276],{"class":124},[49,1699,128],{"class":63},[49,1701,143],{"class":137},[49,1703,161],{"class":63},[49,1705,1706,1708,1710,1712],{"class":51,"line":149},[49,1707,153],{"class":152},[49,1709,128],{"class":63},[49,1711,158],{"class":137},[49,1713,161],{"class":63},[14,1715,1716],{},"gives you a set, not a list.",[14,1718,1719],{},"If you need a list, write:",[39,1721,1723],{"className":41,"code":1722,"language":43,"meta":44,"style":44},"unique_data = list(set(data))\n",[46,1724,1725],{"__ignoreMap":44},[49,1726,1727,1729,1731,1733,1735,1737,1739,1741],{"class":51,"line":52},[49,1728,119],{"class":55},[49,1730,60],{"class":59},[49,1732,125],{"class":124},[49,1734,128],{"class":63},[49,1736,359],{"class":124},[49,1738,128],{"class":63},[49,1740,143],{"class":137},[49,1742,146],{"class":63},[34,1744,1746],{"id":1745},"faq","FAQ",[339,1748,1750],{"id":1749},"what-is-the-easiest-way-to-remove-duplicates-from-a-python-list","What is the easiest way to remove duplicates from a Python list?",[14,1752,171,1753,1756,1757,1760],{},[46,1754,1755],{},"set(your_list)"," if order does not matter. Use ",[46,1758,1759],{},"list(dict.fromkeys(your_list))"," if you want to keep the original order.",[339,1762,1764,1765,1767],{"id":1763},"does-set-keep-the-same-order-as-the-original-list","Does ",[46,1766,1077],{}," keep the same order as the original list?",[14,1769,1770],{},"No. A set removes duplicates, but it does not guarantee the same order as the original list.",[339,1772,1774],{"id":1773},"how-do-i-remove-duplicates-and-keep-order-in-python","How do I remove duplicates and keep order in Python?",[14,1776,171,1777,1780],{},[46,1778,1779],{},"list(dict.fromkeys(data))",". It keeps the first occurrence of each value.",[339,1782,1784,1785,1787],{"id":1783},"can-i-use-set-on-a-list-of-dictionaries","Can I use ",[46,1786,1077],{}," on a list of dictionaries?",[14,1789,1790],{},"No. Dictionaries are mutable and cannot be added to a set directly.",[34,1792,1794],{"id":1793},"see-also","See also",[21,1796,1797,1802,1806,1810,1814,1820],{},[24,1798,1799],{},[198,1800,1801],{"href":200},"How to remove duplicates from a list in Python",[24,1803,1804],{},[198,1805,379],{"href":378},[24,1807,1808],{},[198,1809,384],{"href":383},[24,1811,1812],{},[198,1813,654],{"href":653},[24,1815,1816],{},[198,1817,1819],{"href":1818},"\u002Fhow-to\u002Fhow-to-filter-a-list-in-python\u002F","How to filter a list in Python",[24,1821,1822],{},[198,1823,1303],{"href":1302},[1825,1826,1827],"style",{},"html pre.shiki code .su5hD, html code.shiki .su5hD{--shiki-light:#90A4AE;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .smGrS, html code.shiki .smGrS{--shiki-light:#39ADB5;--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sP7_E, html code.shiki .sP7_E{--shiki-light:#39ADB5;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sjJ54, html code.shiki .sjJ54{--shiki-light:#39ADB5;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .s_sjI, html code.shiki .s_sjI{--shiki-light:#91B859;--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sZMiF, html code.shiki .sZMiF{--shiki-light:#E2931D;--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .slqww, html code.shiki .slqww{--shiki-light:#6182B8;--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sptTA, html code.shiki .sptTA{--shiki-light:#6182B8;--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sutJx, html code.shiki .sutJx{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#6A737D;--shiki-default-font-style:inherit;--shiki-dark:#6A737D;--shiki-dark-font-style:inherit}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sVHd0, html code.shiki .sVHd0{--shiki-light:#39ADB5;--shiki-light-font-style:italic;--shiki-default:#D73A49;--shiki-default-font-style:inherit;--shiki-dark:#F97583;--shiki-dark-font-style:inherit}",{"title":44,"searchDepth":116,"depth":116,"links":1829},[1830,1831,1832,1836,1840,1843,1844,1845,1855,1863],{"id":36,"depth":116,"text":37},{"id":178,"depth":116,"text":179},{"id":204,"depth":116,"text":205,"children":1833},[1834,1835],{"id":341,"depth":149,"text":342},{"id":363,"depth":149,"text":364},{"id":479,"depth":116,"text":480,"children":1837},[1838,1839],{"id":618,"depth":149,"text":619},{"id":635,"depth":149,"text":636},{"id":657,"depth":116,"text":658,"children":1841},[1842],{"id":833,"depth":149,"text":834},{"id":1071,"depth":116,"text":1072},{"id":1121,"depth":116,"text":1122},{"id":1306,"depth":116,"text":1307,"children":1846},[1847,1849,1851,1853],{"id":1313,"depth":149,"text":1848},"Using set() and expecting the original order to stay the same",{"id":1418,"depth":149,"text":1850},"Trying to remove duplicates from a list of dictionaries with set()",{"id":1489,"depth":149,"text":1852},"Not normalizing values like \"Apple\" and \"apple\"",{"id":1646,"depth":149,"text":1854},"Forgetting to convert the result back to a list after using set()",{"id":1745,"depth":116,"text":1746,"children":1856},[1857,1858,1860,1861],{"id":1749,"depth":149,"text":1750},{"id":1763,"depth":149,"text":1859},"Does set() keep the same order as the original list?",{"id":1773,"depth":149,"text":1774},{"id":1783,"depth":149,"text":1862},"Can I use set() on a list of dictionaries?",{"id":1793,"depth":116,"text":1794},"Master python remove duplicates from data example in our comprehensive Python beginner guide.","md",{},"\u002Fexamples\u002Fpython-remove-duplicates-from-data-example",{"title":5,"description":1864},"examples\u002Fpython-remove-duplicates-from-data-example","vVztjqRvKDL0Pft468CGFVqNBQshw-2s30J25Th0rKE",1777585477914]